@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -1,31 +1,16 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
A bundled task is ``scenarios/<name>/`` in the version (instruction, seed
|
|
4
|
-
overlay, ``checks/verify.py``). A session task is the same three things sent
|
|
5
|
-
with the open, for one session, so one published api world serves any number
|
|
6
|
-
of end users each with their own goal, starting rows and pass criteria.
|
|
7
|
-
|
|
8
|
-
This module is the host's half of the contract whose other half is
|
|
9
|
-
``gateway/packages/shared/src/domain/worldTasks.ts`` (the platform validates
|
|
10
|
-
with Zod, the host validates again here: it trusts nothing it did not check).
|
|
11
|
-
Keep the two in step by hand; the limits and the check grammar are the same.
|
|
12
|
-
|
|
13
|
-
* :func:`parse_task_spec` — the spec, checked and normalised.
|
|
14
|
-
* :func:`write_task_dir` — the spec on disk under the session
|
|
15
|
-
(``task.json``, ``instruction.md``, ``checks/verify.py``), where a grade
|
|
16
|
-
and a restart find it.
|
|
17
|
-
* :func:`evaluate_assertions` — the declarative grader, pure Python over
|
|
18
|
-
the state dump, so a check cannot read outside its session.
|
|
19
|
-
"""
|
|
1
|
+
"""Twin of gateway/packages/shared/src/domain/worldTasks.ts, pinned by conformance/world-journal.json; the host
|
|
2
|
+
validates again, trusting nothing it did not check."""
|
|
20
3
|
|
|
21
4
|
from __future__ import annotations
|
|
22
5
|
|
|
23
6
|
import json
|
|
7
|
+
import math
|
|
24
8
|
import re
|
|
25
9
|
from dataclasses import dataclass
|
|
26
10
|
from pathlib import Path
|
|
27
11
|
from typing import Any, Iterable
|
|
28
12
|
|
|
13
|
+
from ..reward_keys import RewardKeys
|
|
29
14
|
from .validation import fail
|
|
30
15
|
|
|
31
16
|
# The caps, as the platform's WORLD_TASK_LIMITS.
|
|
@@ -42,7 +27,6 @@ RUBRIC_CRITERIA = 50
|
|
|
42
27
|
RUBRIC_CRITERION_BYTES = 2 * 1024
|
|
43
28
|
TOOLS = 200
|
|
44
29
|
TOOL_NAME_CHARS = 200
|
|
45
|
-
#: Cross-world verifiers one task may carry.
|
|
46
30
|
VERIFIERS = 32
|
|
47
31
|
|
|
48
32
|
SEED_MODES = ("append", "replace")
|
|
@@ -55,7 +39,6 @@ _TOOL_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]*$")
|
|
|
55
39
|
|
|
56
40
|
TASK_DIR = "task"
|
|
57
41
|
TASK_FILE = "task.json"
|
|
58
|
-
#: Under the task dir, a python verifier's source: ``verifiers/<name>/verify.py``.
|
|
59
42
|
VERIFIERS_DIR = "verifiers"
|
|
60
43
|
|
|
61
44
|
|
|
@@ -70,9 +53,7 @@ def _scalar(value: Any) -> bool:
|
|
|
70
53
|
|
|
71
54
|
@dataclass(frozen=True)
|
|
72
55
|
class TaskSpec:
|
|
73
|
-
"""
|
|
74
|
-
the platform sent, defaults filled, and what is written to ``task.json``
|
|
75
|
-
and compared when an open is retried."""
|
|
56
|
+
"""``raw`` is the normalised spec, written to task.json and compared when an open is retried."""
|
|
76
57
|
|
|
77
58
|
instruction: str
|
|
78
59
|
name: str | None
|
|
@@ -82,8 +63,6 @@ class TaskSpec:
|
|
|
82
63
|
raw: dict
|
|
83
64
|
#: The tools the session may call, or None for every tool the world serves.
|
|
84
65
|
tools: tuple[str, ...] | None = None
|
|
85
|
-
#: Cross-world verifiers anchored on this session: each a grader table
|
|
86
|
-
#: with ``name`` and ``scope`` (see ``TaskGrader.evidence``).
|
|
87
66
|
verifiers: tuple[dict, ...] = ()
|
|
88
67
|
|
|
89
68
|
@property
|
|
@@ -92,59 +71,73 @@ class TaskSpec:
|
|
|
92
71
|
|
|
93
72
|
@property
|
|
94
73
|
def graded_on_platform(self) -> bool:
|
|
95
|
-
"""A rubric task is judged by the platform's inference path, never here."""
|
|
96
74
|
return self.grader_kind == "rubric"
|
|
97
75
|
|
|
98
76
|
def allows_tool(self, tool: str) -> bool:
|
|
99
77
|
return self.tools is None or tool in self.tools
|
|
100
78
|
|
|
101
79
|
def entities(self) -> list[str]:
|
|
102
|
-
"""Every entity the task names, seed rows and assertion checks alike, in
|
|
103
|
-
order of first mention. A world that defines none of a name cannot start
|
|
104
|
-
or grade this task: a seed row has no table, and a check over a table
|
|
105
|
-
that does not exist selects no rows, which an ``absent`` or ``count:
|
|
106
|
-
eq 0`` check would pass on a typo."""
|
|
107
80
|
named: list[str] = []
|
|
108
81
|
if self.seed is not None:
|
|
109
82
|
named.extend(self.seed["rows"])
|
|
110
83
|
if self.grader is not None and self.grader["kind"] == "assertions":
|
|
111
84
|
named.extend(check["entity"] for check in self.grader["checks"])
|
|
112
|
-
# A
|
|
113
|
-
# ones under this session's own alias are this world's to define.
|
|
85
|
+
# A session task is named for its alias: only checks under it name this world's entities.
|
|
114
86
|
own = f"{self.name}."
|
|
115
87
|
for verifier in self.verifiers:
|
|
116
88
|
named.extend(
|
|
117
|
-
check["entity"][len(own) :]
|
|
118
|
-
for check in verifier.get("checks", ())
|
|
119
|
-
if check["entity"].startswith(own)
|
|
89
|
+
check["entity"][len(own) :] for check in verifier.get("checks", ()) if check["entity"].startswith(own)
|
|
120
90
|
)
|
|
121
91
|
return list(dict.fromkeys(named))
|
|
122
92
|
|
|
123
93
|
|
|
124
|
-
#: Keys a state dump carries beside its entities, which a grader reads like an
|
|
125
|
-
#: entity: ``calls`` (the session's call log) and ``snapshot``. A world cannot
|
|
126
|
-
#: declare an entity by either name (the compiler refuses it), so a check on
|
|
127
|
-
#: one is never a typo for a row entity; a seed may not target them.
|
|
128
|
-
RESERVED_STATE_KEYS = frozenset({"calls", "snapshot"})
|
|
129
94
|
CALL_LOG = "calls"
|
|
95
|
+
JOURNAL = "gateway_requests"
|
|
96
|
+
JOURNAL_META = "gateway_journal"
|
|
97
|
+
RESERVED_STATE_KEYS = frozenset({CALL_LOG, "snapshot", JOURNAL})
|
|
130
98
|
DOOR_FIELD = "via"
|
|
99
|
+
WITHHELD_FIELD = "withheld"
|
|
100
|
+
ROW_FIELDS = ("found", "rows")
|
|
101
|
+
ORDER_KEYS = ("before", "after")
|
|
102
|
+
JOURNAL_FIELDS = (
|
|
103
|
+
"seq",
|
|
104
|
+
"id",
|
|
105
|
+
"at",
|
|
106
|
+
"durationMs",
|
|
107
|
+
"via",
|
|
108
|
+
"operation",
|
|
109
|
+
"method",
|
|
110
|
+
"path",
|
|
111
|
+
"rows",
|
|
112
|
+
"found",
|
|
113
|
+
"request.body",
|
|
114
|
+
"request.bodyBytes",
|
|
115
|
+
"request.truncated",
|
|
116
|
+
"request.contentType",
|
|
117
|
+
"response.status",
|
|
118
|
+
"response.body",
|
|
119
|
+
"response.bodyBytes",
|
|
120
|
+
"response.truncated",
|
|
121
|
+
"response.contentType",
|
|
122
|
+
)
|
|
123
|
+
JOURNAL_FIELD_PREFIXES = ("query.", "request.headers.", "request.body.", "response.body.")
|
|
131
124
|
|
|
132
125
|
|
|
133
126
|
class GraderError(Exception):
|
|
134
|
-
"""A grader that
|
|
127
|
+
"""A grader that cannot deliver a grade: the host refuses, the runner records null with the reason, never 0."""
|
|
135
128
|
|
|
136
129
|
|
|
137
130
|
def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
|
|
138
|
-
"""The entities ``spec`` names that ``defined`` (a world contract's) lacks.
|
|
139
|
-
A grader's check on a reserved state key is not one of them."""
|
|
140
131
|
known = set(defined)
|
|
141
132
|
seeded = set((spec.seed or {}).get("rows", {})) if isinstance(spec.seed, dict) else set()
|
|
142
|
-
return [
|
|
143
|
-
|
|
133
|
+
return [
|
|
134
|
+
entity
|
|
135
|
+
for entity in spec.entities()
|
|
136
|
+
if entity not in known and not (entity in RESERVED_STATE_KEYS and entity not in seeded)
|
|
137
|
+
]
|
|
144
138
|
|
|
145
139
|
|
|
146
140
|
def parse_task_spec(value: Any) -> TaskSpec:
|
|
147
|
-
"""The spec checked for shape and size; every refusal names its path."""
|
|
148
141
|
if not isinstance(value, dict):
|
|
149
142
|
fail("task", "a task is a json object", "/task")
|
|
150
143
|
unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata", "verifiers"}
|
|
@@ -163,7 +156,11 @@ def parse_task_spec(value: Any) -> TaskSpec:
|
|
|
163
156
|
tools = _parse_tools(value.get("tools"))
|
|
164
157
|
verifiers = _parse_verifiers(value.get("verifiers"), name)
|
|
165
158
|
if verifiers and grader is not None and grader["kind"] == "rubric":
|
|
166
|
-
fail(
|
|
159
|
+
fail(
|
|
160
|
+
"task",
|
|
161
|
+
"a task with verifiers grades by assertions, python or nothing: a rubric never reaches the host's grade",
|
|
162
|
+
"/task/grader/kind",
|
|
163
|
+
)
|
|
167
164
|
metadata = value.get("metadata")
|
|
168
165
|
if metadata is None:
|
|
169
166
|
metadata = {}
|
|
@@ -200,13 +197,7 @@ _VERIFIER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
|
200
197
|
|
|
201
198
|
|
|
202
199
|
def _parse_verifiers(value: Any, task_name: str | None) -> tuple[dict, ...]:
|
|
203
|
-
"""
|
|
204
|
-
(the grader through :func:`_parse_grader`, so the check grammar and the
|
|
205
|
-
caps are defined once). A scope names two or more distinct aliases; the
|
|
206
|
-
first is this session, so it must be the task's own name — a session task
|
|
207
|
-
is what one alias of a multi-world task opens on, named for that alias.
|
|
208
|
-
Every assertions check names ``<alias>.<entity>`` with the alias in
|
|
209
|
-
scope. Mirrors ``WorldTaskVerifierSchema``."""
|
|
200
|
+
"""Mirrors ``WorldTaskVerifierSchema``."""
|
|
210
201
|
if value is None:
|
|
211
202
|
return ()
|
|
212
203
|
at = "/task/verifiers"
|
|
@@ -227,27 +218,48 @@ def _parse_verifier(row: Any, at: str, task_name: str | None) -> dict:
|
|
|
227
218
|
if not isinstance(name, str) or not _VERIFIER_NAME.match(name) or len(name) > NAME_CHARS:
|
|
228
219
|
fail("task", "a verifier name is letters, digits, _ . -", f"{at}/name")
|
|
229
220
|
scope = row.get("scope")
|
|
230
|
-
if
|
|
221
|
+
if (
|
|
222
|
+
not isinstance(scope, list)
|
|
223
|
+
or len(scope) < 2
|
|
224
|
+
or any(not isinstance(a, str) or not _NAME.match(a) for a in scope)
|
|
225
|
+
):
|
|
231
226
|
fail("task", "a verifier's scope names two or more world aliases", f"{at}/scope")
|
|
232
227
|
if len(set(scope)) != len(scope):
|
|
233
228
|
fail("task", "a verifier's scope lists each alias once", f"{at}/scope")
|
|
229
|
+
reserved = [alias for alias in scope if RewardKeys.reserved(alias)]
|
|
230
|
+
if reserved:
|
|
231
|
+
fail("task", RewardKeys.refusal(reserved[0]), f"{at}/scope")
|
|
234
232
|
if scope[0] != task_name:
|
|
235
|
-
fail(
|
|
233
|
+
fail(
|
|
234
|
+
"task",
|
|
235
|
+
f"verifier '{name}' runs on the session of its first scope alias, so the task's name must be '{scope[0]}'",
|
|
236
|
+
f"{at}/scope",
|
|
237
|
+
)
|
|
236
238
|
grader = _parse_grader({k: v for k, v in row.items() if k not in ("name", "scope")}, at)
|
|
237
239
|
if grader is None or grader["kind"] == "rubric":
|
|
238
240
|
fail("task", "a verifier grades by assertions or python (a rubric judge reads one world)", f"{at}/kind")
|
|
241
|
+
if grader.get("readsJournal"):
|
|
242
|
+
fail("task", CROSS_JOURNAL, f"{at}/readsJournal")
|
|
239
243
|
for j, check in enumerate(grader.get("checks", ())):
|
|
240
|
-
|
|
241
|
-
fail(
|
|
242
|
-
"task",
|
|
243
|
-
f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
|
|
244
|
-
f"{at}/checks/{j}/entity",
|
|
245
|
-
)
|
|
244
|
+
_cross_world_entity(check["entity"], scope, f"{at}/checks/{j}/entity")
|
|
246
245
|
return {"name": name, "scope": list(scope), **grader}
|
|
247
246
|
|
|
248
247
|
|
|
248
|
+
CROSS_JOURNAL = f"a cross-world verifier reads <alias>.{CALL_LOG}; {JOURNAL} checks go on a world's own grader"
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _cross_world_entity(entity: str, scope: list, at: str) -> None:
|
|
252
|
+
if entity.split(".", 1)[0] not in scope or "." not in entity:
|
|
253
|
+
fail(
|
|
254
|
+
"task",
|
|
255
|
+
f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
|
|
256
|
+
at,
|
|
257
|
+
)
|
|
258
|
+
if _on_the_journal(entity):
|
|
259
|
+
fail("task", CROSS_JOURNAL, at)
|
|
260
|
+
|
|
261
|
+
|
|
249
262
|
def verifier_source(task_dir: Path, name: str) -> Path:
|
|
250
|
-
"""Where :func:`write_task_dir` puts a python verifier's ``verify.py``."""
|
|
251
263
|
return Path(task_dir) / VERIFIERS_DIR / name / "verify.py"
|
|
252
264
|
|
|
253
265
|
|
|
@@ -266,21 +278,15 @@ def _parse_tools(tools: Any) -> tuple[str, ...] | None:
|
|
|
266
278
|
return tuple(tools)
|
|
267
279
|
|
|
268
280
|
|
|
269
|
-
def unknown_task_tools(spec: TaskSpec, served: Iterable[str]) -> list[str]:
|
|
270
|
-
"""The tools ``spec`` allows that the world does not serve. A name the
|
|
271
|
-
world never had would make the allow-list smaller than the author meant
|
|
272
|
-
without anyone noticing, so it is refused at open."""
|
|
273
|
-
if spec.tools is None:
|
|
274
|
-
return []
|
|
275
|
-
known = set(served)
|
|
276
|
-
return [tool for tool in spec.tools if tool not in known]
|
|
277
|
-
|
|
278
|
-
|
|
279
281
|
def _parse_seed(seed: Any) -> dict | None:
|
|
280
282
|
if seed is None:
|
|
281
283
|
return None
|
|
282
284
|
if not isinstance(seed, dict) or set(seed) - {"mode", "rows", "redact"}:
|
|
283
|
-
fail(
|
|
285
|
+
fail(
|
|
286
|
+
"task",
|
|
287
|
+
"seed is {mode?: append|replace, redact?: off|apply|refuse, rows: {entity: [row, ...]}}",
|
|
288
|
+
"/task/seed",
|
|
289
|
+
)
|
|
284
290
|
mode = seed.get("mode", "append")
|
|
285
291
|
if mode not in SEED_MODES:
|
|
286
292
|
fail("task", f"seed mode must be one of {', '.join(SEED_MODES)}", "/task/seed/mode")
|
|
@@ -313,24 +319,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
|
313
319
|
fail("task", "grader is a json object with a kind", at)
|
|
314
320
|
kind = grader.get("kind")
|
|
315
321
|
if kind == "python":
|
|
316
|
-
|
|
317
|
-
fail("task", "a python grader is {kind, source, timeoutSeconds?}", at)
|
|
318
|
-
source = grader.get("source")
|
|
319
|
-
if not isinstance(source, str) or not source:
|
|
320
|
-
fail("task", "a python grader needs its verify.py source", f"{at}/source")
|
|
321
|
-
if _bytes(source) > PYTHON_SOURCE_BYTES:
|
|
322
|
-
fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
|
|
323
|
-
out: dict = {"kind": "python", "source": source}
|
|
324
|
-
timeout = grader.get("timeoutSeconds")
|
|
325
|
-
if timeout is not None:
|
|
326
|
-
if (
|
|
327
|
-
isinstance(timeout, bool)
|
|
328
|
-
or not isinstance(timeout, (int, float))
|
|
329
|
-
or not 0 < timeout <= PYTHON_TIMEOUT_S
|
|
330
|
-
):
|
|
331
|
-
fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
|
|
332
|
-
out["timeoutSeconds"] = float(timeout)
|
|
333
|
-
return out
|
|
322
|
+
return _parse_python_grader(grader, at)
|
|
334
323
|
if kind == "assertions":
|
|
335
324
|
if set(grader) - {"kind", "checks"}:
|
|
336
325
|
fail("task", "an assertions grader is {kind, checks}", at)
|
|
@@ -340,9 +329,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
|
340
329
|
if len(checks) > CHECKS:
|
|
341
330
|
fail("task", f"an assertions grader has at most {CHECKS} checks", f"{at}/checks")
|
|
342
331
|
parsed = [_parse_check(check, f"{at}/checks/{i}") for i, check in enumerate(checks)]
|
|
343
|
-
#
|
|
344
|
-
# distinct as a set, so an explicit `check_2` cannot shadow the third
|
|
345
|
-
# check's positional one. Mirrors WorldTaskAssertionsGraderSchema.
|
|
332
|
+
# Mirrors WorldTaskAssertionsGraderSchema.
|
|
346
333
|
seen: set[str] = set()
|
|
347
334
|
for i, check in enumerate(parsed):
|
|
348
335
|
key = check_key(check, i)
|
|
@@ -363,12 +350,50 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
|
363
350
|
if kind == "rubric":
|
|
364
351
|
return _parse_rubric(grader, at)
|
|
365
352
|
fail("task", "grader.kind must be assertions, python or rubric", f"{at}/kind")
|
|
366
|
-
raise AssertionError
|
|
353
|
+
raise AssertionError
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
PYTHON_GRADER_KEYS = frozenset({"kind", "source", "timeoutSeconds", "readsJournal", "acceptsIncompleteJournal"})
|
|
357
|
+
JOURNAL_FLAGS = ("readsJournal", "acceptsIncompleteJournal")
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _parse_python_grader(grader: dict, at: str) -> dict:
|
|
361
|
+
"""``readsJournal`` hands verify.py the session's journal; ``acceptsIncompleteJournal`` grades one that lost
|
|
362
|
+
requests instead of failing closed."""
|
|
363
|
+
if set(grader) - PYTHON_GRADER_KEYS:
|
|
364
|
+
fail("task", "a python grader is {kind, source, timeoutSeconds?, readsJournal?, acceptsIncompleteJournal?}", at)
|
|
365
|
+
source = grader.get("source")
|
|
366
|
+
if not isinstance(source, str) or not source:
|
|
367
|
+
fail("task", "a python grader needs its verify.py source", f"{at}/source")
|
|
368
|
+
if _bytes(source) > PYTHON_SOURCE_BYTES:
|
|
369
|
+
fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
|
|
370
|
+
out: dict = {"kind": "python", "source": source}
|
|
371
|
+
timeout = grader.get("timeoutSeconds")
|
|
372
|
+
if timeout is not None:
|
|
373
|
+
out["timeoutSeconds"] = _timeout(timeout, at)
|
|
374
|
+
for flag in JOURNAL_FLAGS:
|
|
375
|
+
out.update(_flag(grader, flag, at))
|
|
376
|
+
if out.get("acceptsIncompleteJournal") and not out.get("readsJournal"):
|
|
377
|
+
fail("task", "acceptsIncompleteJournal only goes with readsJournal: true", f"{at}/acceptsIncompleteJournal")
|
|
378
|
+
return out
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _timeout(value: Any, at: str) -> float:
|
|
382
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= PYTHON_TIMEOUT_S:
|
|
383
|
+
fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
|
|
384
|
+
return float(value)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _flag(grader: dict, name: str, at: str) -> dict:
|
|
388
|
+
value = grader.get(name)
|
|
389
|
+
if value is None or value is False:
|
|
390
|
+
return {}
|
|
391
|
+
if value is not True:
|
|
392
|
+
fail("task", f"{name} is true or absent", f"{at}/{name}")
|
|
393
|
+
return {name: True}
|
|
367
394
|
|
|
368
395
|
|
|
369
396
|
def _parse_rubric(grader: dict, at: str) -> dict:
|
|
370
|
-
"""An LLM judge's brief. Checked here for shape only: the host stores it
|
|
371
|
-
with the task and never runs it; the platform grades it."""
|
|
372
397
|
if set(grader) - {"kind", "rubric", "criteria", "model"}:
|
|
373
398
|
fail("task", "a rubric grader is {kind, rubric, criteria?, model?}", at)
|
|
374
399
|
rubric = grader.get("rubric")
|
|
@@ -422,81 +447,166 @@ def _parse_rubric(grader: dict, at: str) -> dict:
|
|
|
422
447
|
|
|
423
448
|
|
|
424
449
|
def check_key(check: dict, index: int) -> str:
|
|
425
|
-
"""The reward key of a check: its id, else its position as ``check_<i>``."""
|
|
426
450
|
return check.get("id") or f"check_{index}"
|
|
427
451
|
|
|
428
452
|
|
|
453
|
+
CHECK_KEYS = frozenset({"id", "weight", "entity", "where", "assert", "count", "field", "op", "value", *ORDER_KEYS})
|
|
454
|
+
PREDICATES = ("all", "any")
|
|
455
|
+
|
|
456
|
+
|
|
429
457
|
def _parse_check(check: Any, at: str) -> dict:
|
|
430
458
|
if not isinstance(check, dict):
|
|
431
459
|
fail("task", "a check is a json object", at)
|
|
432
|
-
unknown = set(check) -
|
|
460
|
+
unknown = set(check) - CHECK_KEYS
|
|
433
461
|
if unknown:
|
|
434
462
|
fail("task", f"unknown check field(s): {', '.join(sorted(unknown))}", at)
|
|
435
463
|
out: dict = {}
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
|
|
440
|
-
out["id"] = check_id
|
|
441
|
-
weight = check.get("weight", 1)
|
|
442
|
-
if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0 or weight == float("inf"):
|
|
443
|
-
fail("task", "weight is a positive number", f"{at}/weight")
|
|
444
|
-
out["weight"] = float(weight)
|
|
464
|
+
if check.get("id") is not None:
|
|
465
|
+
out["id"] = _check_id(check["id"], at)
|
|
466
|
+
out["weight"] = _weight(check.get("weight", 1), at)
|
|
445
467
|
entity = check.get("entity")
|
|
446
468
|
if not isinstance(entity, str) or not entity or len(entity) > 120:
|
|
447
469
|
fail("task", "a check names an entity", f"{at}/entity")
|
|
448
470
|
out["entity"] = entity
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
if
|
|
452
|
-
|
|
453
|
-
):
|
|
454
|
-
fail("task", "where is {field: scalar, ...}", f"{at}/where")
|
|
455
|
-
out["where"] = where
|
|
471
|
+
for key in ("where", *ORDER_KEYS):
|
|
472
|
+
selection = _selection(check.get(key), key, f"{at}/{key}")
|
|
473
|
+
if selection is not None:
|
|
474
|
+
out[key] = selection
|
|
456
475
|
assertion = check.get("assert", "exists")
|
|
457
476
|
if assertion not in ASSERTIONS:
|
|
458
477
|
fail("task", f"assert must be one of {', '.join(ASSERTIONS)}", f"{at}/assert")
|
|
459
478
|
out["assert"] = assertion
|
|
479
|
+
out.update(_count(check, assertion, at))
|
|
480
|
+
out.update(_field_test(check, assertion, at))
|
|
481
|
+
_check_reads(out, at)
|
|
482
|
+
return out
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _check_id(value: Any, at: str) -> str:
|
|
486
|
+
if not isinstance(value, str) or not _CHECK_ID.match(value) or len(value) > 120:
|
|
487
|
+
fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
|
|
488
|
+
return value
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _weight(value: Any, at: str) -> float:
|
|
492
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not value > 0 or value == float("inf"):
|
|
493
|
+
fail("task", "weight is a positive number", f"{at}/weight")
|
|
494
|
+
return float(value)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _selection(value: Any, key: str, at: str) -> dict | None:
|
|
498
|
+
if value is None:
|
|
499
|
+
return None
|
|
500
|
+
if not isinstance(value, dict) or any(not isinstance(k, str) or not k or not _scalar(v) for k, v in value.items()):
|
|
501
|
+
fail("task", f"{key} is {{field: scalar, ...}}", at)
|
|
502
|
+
return value
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _count(check: dict, assertion: str, at: str) -> dict:
|
|
460
506
|
count = check.get("count")
|
|
461
|
-
if assertion
|
|
462
|
-
if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
|
|
463
|
-
fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
|
|
464
|
-
for key, bound in count.items():
|
|
465
|
-
if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
|
|
466
|
-
fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
|
|
467
|
-
out["count"] = count
|
|
468
|
-
elif count is not None:
|
|
507
|
+
if assertion != "count" and count is not None:
|
|
469
508
|
fail("task", "count only goes with assert: count", f"{at}/count")
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
509
|
+
if assertion != "count":
|
|
510
|
+
return {}
|
|
511
|
+
if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
|
|
512
|
+
fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
|
|
513
|
+
for key, bound in count.items():
|
|
514
|
+
_count_bound(key, bound, at)
|
|
515
|
+
return {"count": count}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _count_bound(key: str, bound: Any, at: str) -> None:
|
|
519
|
+
if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
|
|
520
|
+
fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _field_test(check: dict, assertion: str, at: str) -> dict:
|
|
524
|
+
field, op, value = check.get("field"), check.get("op", "eq"), check.get("value")
|
|
525
|
+
if assertion not in PREDICATES and (field is not None or "value" in check or "op" in check):
|
|
487
526
|
fail("task", "field/op/value only go with assert: all or any", f"{at}/field")
|
|
488
|
-
|
|
527
|
+
if assertion not in PREDICATES:
|
|
528
|
+
return {}
|
|
529
|
+
if not isinstance(field, str) or not field:
|
|
530
|
+
fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
|
|
531
|
+
if op not in FIELD_OPS:
|
|
532
|
+
fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
|
|
533
|
+
if "value" not in check:
|
|
534
|
+
fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
|
|
535
|
+
_compared(op, value, at)
|
|
536
|
+
return {"field": field, "op": op, "value": value}
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _compared(op: str, value: Any, at: str) -> None:
|
|
540
|
+
if op == "in" and (not isinstance(value, list) or any(not _scalar(v) for v in value)):
|
|
541
|
+
fail("task", "op: in compares against a list of scalars", f"{at}/value")
|
|
542
|
+
if op != "in" and not _scalar(value):
|
|
543
|
+
fail("task", f"op: {op} compares against one scalar", f"{at}/value")
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _on_the_journal(entity: str) -> bool:
|
|
547
|
+
return entity == JOURNAL or entity.endswith(f".{JOURNAL}")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _ordered_log(entity: str) -> bool:
|
|
551
|
+
return _on_the_journal(entity) or entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _check_reads(check: dict, at: str) -> None:
|
|
555
|
+
entity = check["entity"]
|
|
556
|
+
ordered = [key for key in ORDER_KEYS if key in check]
|
|
557
|
+
if ordered and not _ordered_log(entity):
|
|
558
|
+
fail(
|
|
559
|
+
"task",
|
|
560
|
+
f"{ordered[0]} orders requests in time, so it goes on a {JOURNAL} or {CALL_LOG} check, not on {entity}",
|
|
561
|
+
f"{at}/{ordered[0]}",
|
|
562
|
+
)
|
|
563
|
+
if entity == JOURNAL_META or entity.endswith(f".{JOURNAL_META}"):
|
|
564
|
+
fail("task", f"{JOURNAL_META} is the journal's summary, not its rows; check {JOURNAL} instead", f"{at}/entity")
|
|
565
|
+
if _on_the_journal(entity):
|
|
566
|
+
_journal_paths(check, at)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def _journal_paths(check: dict, at: str) -> None:
|
|
570
|
+
named = [(key, path) for key in ("where", *ORDER_KEYS) for path in check.get(key, {})]
|
|
571
|
+
named += [("field", check["field"])] if "field" in check else []
|
|
572
|
+
for key, path in named:
|
|
573
|
+
_journal_path(path, f"{at}/{key}")
|
|
574
|
+
|
|
489
575
|
|
|
576
|
+
def _distance(a: str, b: str) -> int:
|
|
577
|
+
previous = list(range(len(b) + 1))
|
|
578
|
+
for i, left in enumerate(a, 1):
|
|
579
|
+
current = [i]
|
|
580
|
+
for j, right in enumerate(b, 1):
|
|
581
|
+
current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (left != right)))
|
|
582
|
+
previous = current
|
|
583
|
+
return previous[-1]
|
|
490
584
|
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
585
|
+
|
|
586
|
+
def nearest_journal_field(path: str) -> str | None:
|
|
587
|
+
"""Twin of ``nearestJournalField`` in worldJournal.ts."""
|
|
588
|
+
best, best_distance = None, math.inf
|
|
589
|
+
for candidate in (*JOURNAL_FIELDS, *(prefix[:-1] for prefix in JOURNAL_FIELD_PREFIXES)):
|
|
590
|
+
d = _distance(path, candidate)
|
|
591
|
+
if d < best_distance and d <= 3 and d < len(candidate) / 2:
|
|
592
|
+
best, best_distance = candidate, d
|
|
593
|
+
return best
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _journal_path(path: str, at: str) -> None:
|
|
597
|
+
if path in JOURNAL_FIELDS or path.startswith(JOURNAL_FIELD_PREFIXES):
|
|
598
|
+
return
|
|
599
|
+
near = nearest_journal_field(path)
|
|
600
|
+
hint = f"did you mean {near}? " if near else ""
|
|
601
|
+
fail(
|
|
602
|
+
"task",
|
|
603
|
+
f"{JOURNAL} has no field {path}; {hint}match on {', '.join(JOURNAL_FIELDS)} "
|
|
604
|
+
f"or a path under {', '.join(JOURNAL_FIELD_PREFIXES)}",
|
|
605
|
+
at,
|
|
606
|
+
)
|
|
494
607
|
|
|
495
608
|
|
|
496
609
|
def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
|
|
497
|
-
"""The task under the session: ``task.json`` (the whole spec),
|
|
498
|
-
``instruction.md``, and for a python grader ``checks/verify.py`` where
|
|
499
|
-
the bundled layout has it, so the one verifier path runs both."""
|
|
500
610
|
root = Path(session_dir) / TASK_DIR
|
|
501
611
|
root.mkdir(parents=True, exist_ok=True)
|
|
502
612
|
(root / TASK_FILE).write_text(json.dumps(spec.raw, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
@@ -515,139 +625,20 @@ def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
|
|
|
515
625
|
|
|
516
626
|
|
|
517
627
|
def read_task_dir(session_dir: Path) -> TaskSpec | None:
|
|
518
|
-
"""The task a session directory holds, or None for a bundled-task session."""
|
|
519
628
|
path = Path(session_dir) / TASK_DIR / TASK_FILE
|
|
520
629
|
if not path.is_file():
|
|
521
630
|
return None
|
|
522
631
|
return parse_task_spec(json.loads(path.read_text(encoding="utf-8")))
|
|
523
632
|
|
|
524
633
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
def _lookup(row: Any, path: str) -> tuple[bool, Any]:
|
|
531
|
-
"""``(found, value)`` of a dotted path into a row."""
|
|
532
|
-
current = row
|
|
533
|
-
for part in path.split("."):
|
|
534
|
-
if not isinstance(current, dict) or part not in current:
|
|
535
|
-
return False, None
|
|
536
|
-
current = current[part]
|
|
537
|
-
return True, current
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
def _same(actual: Any, expected: Any) -> bool:
|
|
541
|
-
"""JSON equality: ``true`` is not ``1``, as it is not in the schema."""
|
|
542
|
-
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
543
|
-
return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
|
|
544
|
-
return actual == expected
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
def _matches_where(row: dict, where: dict | None) -> bool:
|
|
548
|
-
for path, expected in (where or {}).items():
|
|
549
|
-
found, actual = _lookup(row, path)
|
|
550
|
-
if not found or not _same(actual, expected):
|
|
551
|
-
return False
|
|
552
|
-
return True
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
def _ordered(actual: Any, expected: Any) -> bool:
|
|
556
|
-
"""Whether the two compare with ``<``: both numbers (not bools) or both strings."""
|
|
557
|
-
numbers = (int, float)
|
|
558
|
-
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
559
|
-
return False
|
|
560
|
-
if isinstance(actual, numbers) and isinstance(expected, numbers):
|
|
561
|
-
return True
|
|
562
|
-
return isinstance(actual, str) and isinstance(expected, str)
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
def _predicate(row: dict, field: str, op: str, expected: Any) -> bool:
|
|
566
|
-
found, actual = _lookup(row, field)
|
|
567
|
-
if op == "eq":
|
|
568
|
-
return found and _same(actual, expected)
|
|
569
|
-
if op == "neq":
|
|
570
|
-
return not found or not _same(actual, expected)
|
|
571
|
-
if op == "in":
|
|
572
|
-
return found and any(_same(actual, option) for option in expected)
|
|
573
|
-
if op == "contains":
|
|
574
|
-
if not found:
|
|
575
|
-
return False
|
|
576
|
-
if isinstance(actual, str):
|
|
577
|
-
return isinstance(expected, str) and expected in actual
|
|
578
|
-
if isinstance(actual, list):
|
|
579
|
-
return any(_same(item, expected) for item in actual)
|
|
580
|
-
return False
|
|
581
|
-
if not found or not _ordered(actual, expected):
|
|
582
|
-
return False
|
|
583
|
-
if op == "gt":
|
|
584
|
-
return actual > expected
|
|
585
|
-
if op == "gte":
|
|
586
|
-
return actual >= expected
|
|
587
|
-
if op == "lt":
|
|
588
|
-
return actual < expected
|
|
589
|
-
return actual <= expected # lte
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
def _rows_of(state: dict, entity: str) -> list[dict]:
|
|
593
|
-
"""The rows of one entity from a state dump: ``{entity: {key: row}}`` (the
|
|
594
|
-
verifier's shape) or ``{entity: [row, ...]}``."""
|
|
595
|
-
held = state.get(entity)
|
|
596
|
-
if isinstance(held, dict):
|
|
597
|
-
return [row for row in held.values() if isinstance(row, dict)]
|
|
598
|
-
if isinstance(held, list):
|
|
599
|
-
return [row for row in held if isinstance(row, dict)]
|
|
600
|
-
return []
|
|
634
|
+
def _on_the_call_log(check: dict) -> bool:
|
|
635
|
+
entity = check["entity"]
|
|
636
|
+
return entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
|
|
601
637
|
|
|
602
638
|
|
|
603
|
-
def
|
|
604
|
-
|
|
605
|
-
on_call_log = entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
|
|
606
|
-
return on_call_log and (check.get("field") == DOOR_FIELD or DOOR_FIELD in (check.get("where") or {}))
|
|
639
|
+
def _names(check: dict, field: str) -> bool:
|
|
640
|
+
return check.get("field") == field or field in (check.get("where") or {})
|
|
607
641
|
|
|
608
642
|
|
|
609
|
-
def
|
|
610
|
-
|
|
611
|
-
would select nothing (``absent`` passes) or miss the key (``neq`` passes):
|
|
612
|
-
ungraded."""
|
|
613
|
-
if not _names_the_door(check):
|
|
614
|
-
return
|
|
615
|
-
if check["entity"] not in state or any(DOOR_FIELD not in row for row in rows):
|
|
616
|
-
raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
def evaluate_check(check: dict, state: dict) -> bool:
|
|
620
|
-
rows = _rows_of(state, check["entity"])
|
|
621
|
-
_require_the_door(check, state, rows)
|
|
622
|
-
selected = [row for row in rows if _matches_where(row, check.get("where"))]
|
|
623
|
-
assertion = check["assert"]
|
|
624
|
-
if assertion == "exists":
|
|
625
|
-
return bool(selected)
|
|
626
|
-
if assertion == "absent":
|
|
627
|
-
return not selected
|
|
628
|
-
if assertion == "count":
|
|
629
|
-
bound = check["count"]
|
|
630
|
-
n = len(selected)
|
|
631
|
-
return all(
|
|
632
|
-
(key != "eq" or n == value) and (key != "gte" or n >= value) and (key != "lte" or n <= value)
|
|
633
|
-
for key, value in bound.items()
|
|
634
|
-
)
|
|
635
|
-
tests = (_predicate(row, check["field"], check["op"], check["value"]) for row in selected)
|
|
636
|
-
if assertion == "all":
|
|
637
|
-
return bool(selected) and all(tests)
|
|
638
|
-
return any(tests) # any
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
def evaluate_assertions(checks: list[dict], state: dict) -> dict:
|
|
642
|
-
"""The grade of an assertions grader over a state dump: the weighted share
|
|
643
|
-
of checks that passed as ``reward``, each check's 0/1 under its id (or its
|
|
644
|
-
position) in ``rewards``. The runner's shape, as ``verify.py`` prints it."""
|
|
645
|
-
rewards: dict[str, float] = {}
|
|
646
|
-
total = 0.0
|
|
647
|
-
passed = 0.0
|
|
648
|
-
for i, check in enumerate(checks):
|
|
649
|
-
ok = evaluate_check(check, state)
|
|
650
|
-
rewards[check_key(check, i)] = 1.0 if ok else 0.0
|
|
651
|
-
total += check["weight"]
|
|
652
|
-
passed += check["weight"] if ok else 0.0
|
|
653
|
-
return {"reward": (passed / total) if total else None, "rewards": rewards}
|
|
643
|
+
def _names_the_door(check: dict) -> bool:
|
|
644
|
+
return _on_the_call_log(check) and _names(check, DOOR_FIELD)
|