@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -1,22 +1,5 @@
1
- """Session tasks: a task supplied at open instead of read from the bundle.
2
-
3
- A bundled task is ``scenarios/<name>/`` in the version (instruction, seed
4
- overlay, ``checks/verify.py``). A session task is the same three things sent
5
- with the open, for one session, so one published api world serves any number
6
- of end users each with their own goal, starting rows and pass criteria.
7
-
8
- This module is the host's half of the contract whose other half is
9
- ``gateway/packages/shared/src/domain/worldTasks.ts`` (the platform validates
10
- with Zod, the host validates again here: it trusts nothing it did not check).
11
- Keep the two in step by hand; the limits and the check grammar are the same.
12
-
13
- * :func:`parse_task_spec` — the spec, checked and normalised.
14
- * :func:`write_task_dir` — the spec on disk under the session
15
- (``task.json``, ``instruction.md``, ``checks/verify.py``), where a grade
16
- and a restart find it.
17
- * :func:`evaluate_assertions` — the declarative grader, pure Python over
18
- the state dump, so a check cannot read outside its session.
19
- """
1
+ """Twin of gateway/packages/shared/src/domain/worldTasks.ts, pinned by conformance/world-journal.json; the host
2
+ validates again, trusting nothing it did not check."""
20
3
 
21
4
  from __future__ import annotations
22
5
 
@@ -44,7 +27,6 @@ RUBRIC_CRITERIA = 50
44
27
  RUBRIC_CRITERION_BYTES = 2 * 1024
45
28
  TOOLS = 200
46
29
  TOOL_NAME_CHARS = 200
47
- #: Cross-world verifiers one task may carry.
48
30
  VERIFIERS = 32
49
31
 
50
32
  SEED_MODES = ("append", "replace")
@@ -57,7 +39,6 @@ _TOOL_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]*$")
57
39
 
58
40
  TASK_DIR = "task"
59
41
  TASK_FILE = "task.json"
60
- #: Under the task dir, a python verifier's source: ``verifiers/<name>/verify.py``.
61
42
  VERIFIERS_DIR = "verifiers"
62
43
 
63
44
 
@@ -72,9 +53,7 @@ def _scalar(value: Any) -> bool:
72
53
 
73
54
  @dataclass(frozen=True)
74
55
  class TaskSpec:
75
- """A validated session task. ``raw`` is the normalised spec as a dict: what
76
- the platform sent, defaults filled, and what is written to ``task.json``
77
- and compared when an open is retried."""
56
+ """``raw`` is the normalised spec, written to task.json and compared when an open is retried."""
78
57
 
79
58
  instruction: str
80
59
  name: str | None
@@ -84,8 +63,6 @@ class TaskSpec:
84
63
  raw: dict
85
64
  #: The tools the session may call, or None for every tool the world serves.
86
65
  tools: tuple[str, ...] | None = None
87
- #: Cross-world verifiers anchored on this session: each a grader table
88
- #: with ``name`` and ``scope`` (see ``TaskGrader.evidence``).
89
66
  verifiers: tuple[dict, ...] = ()
90
67
 
91
68
  @property
@@ -94,25 +71,18 @@ class TaskSpec:
94
71
 
95
72
  @property
96
73
  def graded_on_platform(self) -> bool:
97
- """A rubric task is judged by the platform's inference path, never here."""
98
74
  return self.grader_kind == "rubric"
99
75
 
100
76
  def allows_tool(self, tool: str) -> bool:
101
77
  return self.tools is None or tool in self.tools
102
78
 
103
79
  def entities(self) -> list[str]:
104
- """Every entity the task names, seed rows and assertion checks alike, in
105
- order of first mention. A world that defines none of a name cannot start
106
- or grade this task: a seed row has no table, and a check over a table
107
- that does not exist selects no rows, which an ``absent`` or ``count:
108
- eq 0`` check would pass on a typo."""
109
80
  named: list[str] = []
110
81
  if self.seed is not None:
111
82
  named.extend(self.seed["rows"])
112
83
  if self.grader is not None and self.grader["kind"] == "assertions":
113
84
  named.extend(check["entity"] for check in self.grader["checks"])
114
- # A verifier's checks name other worlds' entities as well; only the
115
- # ones under this session's own alias are this world's to define.
85
+ # A session task is named for its alias: only checks under it name this world's entities.
116
86
  own = f"{self.name}."
117
87
  for verifier in self.verifiers:
118
88
  named.extend(
@@ -121,24 +91,43 @@ class TaskSpec:
121
91
  return list(dict.fromkeys(named))
122
92
 
123
93
 
124
- #: Keys a state dump carries beside its entities, which a grader reads like an
125
- #: entity: ``calls`` (the session's call log) and ``snapshot``. A world cannot
126
- #: declare an entity by either name (the compiler refuses it), so a check on
127
- #: one is never a typo for a row entity; a seed may not target them.
128
- RESERVED_STATE_KEYS = frozenset({"calls", "snapshot"})
129
94
  CALL_LOG = "calls"
95
+ JOURNAL = "gateway_requests"
96
+ JOURNAL_META = "gateway_journal"
97
+ RESERVED_STATE_KEYS = frozenset({CALL_LOG, "snapshot", JOURNAL})
130
98
  DOOR_FIELD = "via"
131
99
  WITHHELD_FIELD = "withheld"
132
100
  ROW_FIELDS = ("found", "rows")
101
+ ORDER_KEYS = ("before", "after")
102
+ JOURNAL_FIELDS = (
103
+ "seq",
104
+ "id",
105
+ "at",
106
+ "durationMs",
107
+ "via",
108
+ "operation",
109
+ "method",
110
+ "path",
111
+ "rows",
112
+ "found",
113
+ "request.body",
114
+ "request.bodyBytes",
115
+ "request.truncated",
116
+ "request.contentType",
117
+ "response.status",
118
+ "response.body",
119
+ "response.bodyBytes",
120
+ "response.truncated",
121
+ "response.contentType",
122
+ )
123
+ JOURNAL_FIELD_PREFIXES = ("query.", "request.headers.", "request.body.", "response.body.")
133
124
 
134
125
 
135
126
  class GraderError(Exception):
136
- """A grader that could not deliver a grade, with the reason a reader acts on."""
127
+ """A grader that cannot deliver a grade: the host refuses, the runner records null with the reason, never 0."""
137
128
 
138
129
 
139
130
  def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
140
- """The entities ``spec`` names that ``defined`` (a world contract's) lacks.
141
- A grader's check on a reserved state key is not one of them."""
142
131
  known = set(defined)
143
132
  seeded = set((spec.seed or {}).get("rows", {})) if isinstance(spec.seed, dict) else set()
144
133
  return [
@@ -149,7 +138,6 @@ def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
149
138
 
150
139
 
151
140
  def parse_task_spec(value: Any) -> TaskSpec:
152
- """The spec checked for shape and size; every refusal names its path."""
153
141
  if not isinstance(value, dict):
154
142
  fail("task", "a task is a json object", "/task")
155
143
  unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata", "verifiers"}
@@ -209,13 +197,7 @@ _VERIFIER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
209
197
 
210
198
 
211
199
  def _parse_verifiers(value: Any, task_name: str | None) -> tuple[dict, ...]:
212
- """``verifiers``: cross-world graders, each ``{name, scope, ...grader}``
213
- (the grader through :func:`_parse_grader`, so the check grammar and the
214
- caps are defined once). A scope names two or more distinct aliases; the
215
- first is this session, so it must be the task's own name — a session task
216
- is what one alias of a multi-world task opens on, named for that alias.
217
- Every assertions check names ``<alias>.<entity>`` with the alias in
218
- scope. Mirrors ``WorldTaskVerifierSchema``."""
200
+ """Mirrors ``WorldTaskVerifierSchema``."""
219
201
  if value is None:
220
202
  return ()
221
203
  at = "/task/verifiers"
@@ -256,18 +238,28 @@ def _parse_verifier(row: Any, at: str, task_name: str | None) -> dict:
256
238
  grader = _parse_grader({k: v for k, v in row.items() if k not in ("name", "scope")}, at)
257
239
  if grader is None or grader["kind"] == "rubric":
258
240
  fail("task", "a verifier grades by assertions or python (a rubric judge reads one world)", f"{at}/kind")
241
+ if grader.get("readsJournal"):
242
+ fail("task", CROSS_JOURNAL, f"{at}/readsJournal")
259
243
  for j, check in enumerate(grader.get("checks", ())):
260
- if check["entity"].split(".", 1)[0] not in scope or "." not in check["entity"]:
261
- fail(
262
- "task",
263
- f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
264
- f"{at}/checks/{j}/entity",
265
- )
244
+ _cross_world_entity(check["entity"], scope, f"{at}/checks/{j}/entity")
266
245
  return {"name": name, "scope": list(scope), **grader}
267
246
 
268
247
 
248
+ CROSS_JOURNAL = f"a cross-world verifier reads <alias>.{CALL_LOG}; {JOURNAL} checks go on a world's own grader"
249
+
250
+
251
+ def _cross_world_entity(entity: str, scope: list, at: str) -> None:
252
+ if entity.split(".", 1)[0] not in scope or "." not in entity:
253
+ fail(
254
+ "task",
255
+ f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
256
+ at,
257
+ )
258
+ if _on_the_journal(entity):
259
+ fail("task", CROSS_JOURNAL, at)
260
+
261
+
269
262
  def verifier_source(task_dir: Path, name: str) -> Path:
270
- """Where :func:`write_task_dir` puts a python verifier's ``verify.py``."""
271
263
  return Path(task_dir) / VERIFIERS_DIR / name / "verify.py"
272
264
 
273
265
 
@@ -327,24 +319,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
327
319
  fail("task", "grader is a json object with a kind", at)
328
320
  kind = grader.get("kind")
329
321
  if kind == "python":
330
- if set(grader) - {"kind", "source", "timeoutSeconds"}:
331
- fail("task", "a python grader is {kind, source, timeoutSeconds?}", at)
332
- source = grader.get("source")
333
- if not isinstance(source, str) or not source:
334
- fail("task", "a python grader needs its verify.py source", f"{at}/source")
335
- if _bytes(source) > PYTHON_SOURCE_BYTES:
336
- fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
337
- out: dict = {"kind": "python", "source": source}
338
- timeout = grader.get("timeoutSeconds")
339
- if timeout is not None:
340
- if (
341
- isinstance(timeout, bool)
342
- or not isinstance(timeout, (int, float))
343
- or not 0 < timeout <= PYTHON_TIMEOUT_S
344
- ):
345
- fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
346
- out["timeoutSeconds"] = float(timeout)
347
- return out
322
+ return _parse_python_grader(grader, at)
348
323
  if kind == "assertions":
349
324
  if set(grader) - {"kind", "checks"}:
350
325
  fail("task", "an assertions grader is {kind, checks}", at)
@@ -354,9 +329,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
354
329
  if len(checks) > CHECKS:
355
330
  fail("task", f"an assertions grader has at most {CHECKS} checks", f"{at}/checks")
356
331
  parsed = [_parse_check(check, f"{at}/checks/{i}") for i, check in enumerate(checks)]
357
- # Every check owns one reward key, its id or its position; the keys are
358
- # distinct as a set, so an explicit `check_2` cannot shadow the third
359
- # check's positional one. Mirrors WorldTaskAssertionsGraderSchema.
332
+ # Mirrors WorldTaskAssertionsGraderSchema.
360
333
  seen: set[str] = set()
361
334
  for i, check in enumerate(parsed):
362
335
  key = check_key(check, i)
@@ -377,12 +350,50 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
377
350
  if kind == "rubric":
378
351
  return _parse_rubric(grader, at)
379
352
  fail("task", "grader.kind must be assertions, python or rubric", f"{at}/kind")
380
- raise AssertionError # fail() never returns
353
+ raise AssertionError
354
+
355
+
356
+ PYTHON_GRADER_KEYS = frozenset({"kind", "source", "timeoutSeconds", "readsJournal", "acceptsIncompleteJournal"})
357
+ JOURNAL_FLAGS = ("readsJournal", "acceptsIncompleteJournal")
358
+
359
+
360
+ def _parse_python_grader(grader: dict, at: str) -> dict:
361
+ """``readsJournal`` hands verify.py the session's journal; ``acceptsIncompleteJournal`` grades one that lost
362
+ requests instead of failing closed."""
363
+ if set(grader) - PYTHON_GRADER_KEYS:
364
+ fail("task", "a python grader is {kind, source, timeoutSeconds?, readsJournal?, acceptsIncompleteJournal?}", at)
365
+ source = grader.get("source")
366
+ if not isinstance(source, str) or not source:
367
+ fail("task", "a python grader needs its verify.py source", f"{at}/source")
368
+ if _bytes(source) > PYTHON_SOURCE_BYTES:
369
+ fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
370
+ out: dict = {"kind": "python", "source": source}
371
+ timeout = grader.get("timeoutSeconds")
372
+ if timeout is not None:
373
+ out["timeoutSeconds"] = _timeout(timeout, at)
374
+ for flag in JOURNAL_FLAGS:
375
+ out.update(_flag(grader, flag, at))
376
+ if out.get("acceptsIncompleteJournal") and not out.get("readsJournal"):
377
+ fail("task", "acceptsIncompleteJournal only goes with readsJournal: true", f"{at}/acceptsIncompleteJournal")
378
+ return out
379
+
380
+
381
+ def _timeout(value: Any, at: str) -> float:
382
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= PYTHON_TIMEOUT_S:
383
+ fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
384
+ return float(value)
385
+
386
+
387
+ def _flag(grader: dict, name: str, at: str) -> dict:
388
+ value = grader.get(name)
389
+ if value is None or value is False:
390
+ return {}
391
+ if value is not True:
392
+ fail("task", f"{name} is true or absent", f"{at}/{name}")
393
+ return {name: True}
381
394
 
382
395
 
383
396
  def _parse_rubric(grader: dict, at: str) -> dict:
384
- """An LLM judge's brief. Checked here for shape only: the host stores it
385
- with the task and never runs it; the platform grades it."""
386
397
  if set(grader) - {"kind", "rubric", "criteria", "model"}:
387
398
  fail("task", "a rubric grader is {kind, rubric, criteria?, model?}", at)
388
399
  rubric = grader.get("rubric")
@@ -436,76 +447,166 @@ def _parse_rubric(grader: dict, at: str) -> dict:
436
447
 
437
448
 
438
449
  def check_key(check: dict, index: int) -> str:
439
- """The reward key of a check: its id, else its position as ``check_<i>``."""
440
450
  return check.get("id") or f"check_{index}"
441
451
 
442
452
 
453
+ CHECK_KEYS = frozenset({"id", "weight", "entity", "where", "assert", "count", "field", "op", "value", *ORDER_KEYS})
454
+ PREDICATES = ("all", "any")
455
+
456
+
443
457
  def _parse_check(check: Any, at: str) -> dict:
444
458
  if not isinstance(check, dict):
445
459
  fail("task", "a check is a json object", at)
446
- unknown = set(check) - {"id", "weight", "entity", "where", "assert", "count", "field", "op", "value"}
460
+ unknown = set(check) - CHECK_KEYS
447
461
  if unknown:
448
462
  fail("task", f"unknown check field(s): {', '.join(sorted(unknown))}", at)
449
463
  out: dict = {}
450
- check_id = check.get("id")
451
- if check_id is not None:
452
- if not isinstance(check_id, str) or not _CHECK_ID.match(check_id) or len(check_id) > 120:
453
- fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
454
- out["id"] = check_id
455
- weight = check.get("weight", 1)
456
- if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0 or weight == float("inf"):
457
- fail("task", "weight is a positive number", f"{at}/weight")
458
- out["weight"] = float(weight)
464
+ if check.get("id") is not None:
465
+ out["id"] = _check_id(check["id"], at)
466
+ out["weight"] = _weight(check.get("weight", 1), at)
459
467
  entity = check.get("entity")
460
468
  if not isinstance(entity, str) or not entity or len(entity) > 120:
461
469
  fail("task", "a check names an entity", f"{at}/entity")
462
470
  out["entity"] = entity
463
- where = check.get("where")
464
- if where is not None:
465
- if not isinstance(where, dict) or any(
466
- not isinstance(k, str) or not k or not _scalar(v) for k, v in where.items()
467
- ):
468
- fail("task", "where is {field: scalar, ...}", f"{at}/where")
469
- out["where"] = where
471
+ for key in ("where", *ORDER_KEYS):
472
+ selection = _selection(check.get(key), key, f"{at}/{key}")
473
+ if selection is not None:
474
+ out[key] = selection
470
475
  assertion = check.get("assert", "exists")
471
476
  if assertion not in ASSERTIONS:
472
477
  fail("task", f"assert must be one of {', '.join(ASSERTIONS)}", f"{at}/assert")
473
478
  out["assert"] = assertion
479
+ out.update(_count(check, assertion, at))
480
+ out.update(_field_test(check, assertion, at))
481
+ _check_reads(out, at)
482
+ return out
483
+
484
+
485
+ def _check_id(value: Any, at: str) -> str:
486
+ if not isinstance(value, str) or not _CHECK_ID.match(value) or len(value) > 120:
487
+ fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
488
+ return value
489
+
490
+
491
+ def _weight(value: Any, at: str) -> float:
492
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not value > 0 or value == float("inf"):
493
+ fail("task", "weight is a positive number", f"{at}/weight")
494
+ return float(value)
495
+
496
+
497
+ def _selection(value: Any, key: str, at: str) -> dict | None:
498
+ if value is None:
499
+ return None
500
+ if not isinstance(value, dict) or any(not isinstance(k, str) or not k or not _scalar(v) for k, v in value.items()):
501
+ fail("task", f"{key} is {{field: scalar, ...}}", at)
502
+ return value
503
+
504
+
505
+ def _count(check: dict, assertion: str, at: str) -> dict:
474
506
  count = check.get("count")
475
- if assertion == "count":
476
- if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
477
- fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
478
- for key, bound in count.items():
479
- if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
480
- fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
481
- out["count"] = count
482
- elif count is not None:
507
+ if assertion != "count" and count is not None:
483
508
  fail("task", "count only goes with assert: count", f"{at}/count")
484
- field = check.get("field")
485
- op = check.get("op", "eq")
486
- value = check.get("value")
487
- if assertion in ("all", "any"):
488
- if not isinstance(field, str) or not field:
489
- fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
490
- if op not in FIELD_OPS:
491
- fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
492
- if "value" not in check:
493
- fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
494
- if op == "in":
495
- if not isinstance(value, list) or any(not _scalar(v) for v in value):
496
- fail("task", "op: in compares against a list of scalars", f"{at}/value")
497
- elif not _scalar(value):
498
- fail("task", f"op: {op} compares against one scalar", f"{at}/value")
499
- out["field"], out["op"], out["value"] = field, op, value
500
- elif field is not None or "value" in check or "op" in check:
509
+ if assertion != "count":
510
+ return {}
511
+ if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
512
+ fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
513
+ for key, bound in count.items():
514
+ _count_bound(key, bound, at)
515
+ return {"count": count}
516
+
517
+
518
+ def _count_bound(key: str, bound: Any, at: str) -> None:
519
+ if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
520
+ fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
521
+
522
+
523
+ def _field_test(check: dict, assertion: str, at: str) -> dict:
524
+ field, op, value = check.get("field"), check.get("op", "eq"), check.get("value")
525
+ if assertion not in PREDICATES and (field is not None or "value" in check or "op" in check):
501
526
  fail("task", "field/op/value only go with assert: all or any", f"{at}/field")
502
- return out
527
+ if assertion not in PREDICATES:
528
+ return {}
529
+ if not isinstance(field, str) or not field:
530
+ fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
531
+ if op not in FIELD_OPS:
532
+ fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
533
+ if "value" not in check:
534
+ fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
535
+ _compared(op, value, at)
536
+ return {"field": field, "op": op, "value": value}
537
+
538
+
539
+ def _compared(op: str, value: Any, at: str) -> None:
540
+ if op == "in" and (not isinstance(value, list) or any(not _scalar(v) for v in value)):
541
+ fail("task", "op: in compares against a list of scalars", f"{at}/value")
542
+ if op != "in" and not _scalar(value):
543
+ fail("task", f"op: {op} compares against one scalar", f"{at}/value")
544
+
545
+
546
+ def _on_the_journal(entity: str) -> bool:
547
+ return entity == JOURNAL or entity.endswith(f".{JOURNAL}")
548
+
549
+
550
+ def _ordered_log(entity: str) -> bool:
551
+ return _on_the_journal(entity) or entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
552
+
553
+
554
+ def _check_reads(check: dict, at: str) -> None:
555
+ entity = check["entity"]
556
+ ordered = [key for key in ORDER_KEYS if key in check]
557
+ if ordered and not _ordered_log(entity):
558
+ fail(
559
+ "task",
560
+ f"{ordered[0]} orders requests in time, so it goes on a {JOURNAL} or {CALL_LOG} check, not on {entity}",
561
+ f"{at}/{ordered[0]}",
562
+ )
563
+ if entity == JOURNAL_META or entity.endswith(f".{JOURNAL_META}"):
564
+ fail("task", f"{JOURNAL_META} is the journal's summary, not its rows; check {JOURNAL} instead", f"{at}/entity")
565
+ if _on_the_journal(entity):
566
+ _journal_paths(check, at)
567
+
568
+
569
+ def _journal_paths(check: dict, at: str) -> None:
570
+ named = [(key, path) for key in ("where", *ORDER_KEYS) for path in check.get(key, {})]
571
+ named += [("field", check["field"])] if "field" in check else []
572
+ for key, path in named:
573
+ _journal_path(path, f"{at}/{key}")
574
+
575
+
576
+ def _distance(a: str, b: str) -> int:
577
+ previous = list(range(len(b) + 1))
578
+ for i, left in enumerate(a, 1):
579
+ current = [i]
580
+ for j, right in enumerate(b, 1):
581
+ current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (left != right)))
582
+ previous = current
583
+ return previous[-1]
584
+
585
+
586
+ def nearest_journal_field(path: str) -> str | None:
587
+ """Twin of ``nearestJournalField`` in worldJournal.ts."""
588
+ best, best_distance = None, math.inf
589
+ for candidate in (*JOURNAL_FIELDS, *(prefix[:-1] for prefix in JOURNAL_FIELD_PREFIXES)):
590
+ d = _distance(path, candidate)
591
+ if d < best_distance and d <= 3 and d < len(candidate) / 2:
592
+ best, best_distance = candidate, d
593
+ return best
594
+
595
+
596
+ def _journal_path(path: str, at: str) -> None:
597
+ if path in JOURNAL_FIELDS or path.startswith(JOURNAL_FIELD_PREFIXES):
598
+ return
599
+ near = nearest_journal_field(path)
600
+ hint = f"did you mean {near}? " if near else ""
601
+ fail(
602
+ "task",
603
+ f"{JOURNAL} has no field {path}; {hint}match on {', '.join(JOURNAL_FIELDS)} "
604
+ f"or a path under {', '.join(JOURNAL_FIELD_PREFIXES)}",
605
+ at,
606
+ )
503
607
 
504
608
 
505
609
  def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
506
- """The task under the session: ``task.json`` (the whole spec),
507
- ``instruction.md``, and for a python grader ``checks/verify.py`` where
508
- the bundled layout has it, so the one verifier path runs both."""
509
610
  root = Path(session_dir) / TASK_DIR
510
611
  root.mkdir(parents=True, exist_ok=True)
511
612
  (root / TASK_FILE).write_text(json.dumps(spec.raw, indent=2, sort_keys=True) + "\n", encoding="utf-8")
@@ -524,86 +625,12 @@ def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
524
625
 
525
626
 
526
627
  def read_task_dir(session_dir: Path) -> TaskSpec | None:
527
- """The task a session directory holds, or None for a bundled-task session."""
528
628
  path = Path(session_dir) / TASK_DIR / TASK_FILE
529
629
  if not path.is_file():
530
630
  return None
531
631
  return parse_task_spec(json.loads(path.read_text(encoding="utf-8")))
532
632
 
533
633
 
534
- def _lookup(row: Any, path: str) -> tuple[bool, Any]:
535
- """``(found, value)`` of a dotted path into a row."""
536
- current = row
537
- for part in path.split("."):
538
- if not isinstance(current, dict) or part not in current:
539
- return False, None
540
- current = current[part]
541
- return True, current
542
-
543
-
544
- def _same(actual: Any, expected: Any) -> bool:
545
- """JSON equality: ``true`` is not ``1``, as it is not in the schema."""
546
- if isinstance(actual, bool) or isinstance(expected, bool):
547
- return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
548
- return actual == expected
549
-
550
-
551
- def _matches_where(row: dict, where: dict | None) -> bool:
552
- for path, expected in (where or {}).items():
553
- found, actual = _lookup(row, path)
554
- if not found or not _same(actual, expected):
555
- return False
556
- return True
557
-
558
-
559
- def _ordered(actual: Any, expected: Any) -> bool:
560
- """Whether the two compare with ``<``: both numbers (not bools) or both strings."""
561
- numbers = (int, float)
562
- if isinstance(actual, bool) or isinstance(expected, bool):
563
- return False
564
- if isinstance(actual, numbers) and isinstance(expected, numbers):
565
- return True
566
- return isinstance(actual, str) and isinstance(expected, str)
567
-
568
-
569
- def _predicate(row: dict, field: str, op: str, expected: Any) -> bool:
570
- found, actual = _lookup(row, field)
571
- if op == "eq":
572
- return found and _same(actual, expected)
573
- if op == "neq":
574
- return not found or not _same(actual, expected)
575
- if op == "in":
576
- return found and any(_same(actual, option) for option in expected)
577
- if op == "contains":
578
- if not found:
579
- return False
580
- if isinstance(actual, str):
581
- return isinstance(expected, str) and expected in actual
582
- if isinstance(actual, list):
583
- return any(_same(item, expected) for item in actual)
584
- return False
585
- if not found or not _ordered(actual, expected):
586
- return False
587
- if op == "gt":
588
- return actual > expected
589
- if op == "gte":
590
- return actual >= expected
591
- if op == "lt":
592
- return actual < expected
593
- return actual <= expected
594
-
595
-
596
- def _rows_of(state: dict, entity: str) -> list[dict]:
597
- """The rows of one entity from a state dump: ``{entity: {key: row}}`` (the
598
- verifier's shape) or ``{entity: [row, ...]}``."""
599
- held = state.get(entity)
600
- if isinstance(held, dict):
601
- return [row for row in held.values() if isinstance(row, dict)]
602
- if isinstance(held, list):
603
- return [row for row in held if isinstance(row, dict)]
604
- return []
605
-
606
-
607
634
  def _on_the_call_log(check: dict) -> bool:
608
635
  entity = check["entity"]
609
636
  return entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
@@ -615,124 +642,3 @@ def _names(check: dict, field: str) -> bool:
615
642
 
616
643
  def _names_the_door(check: dict) -> bool:
617
644
  return _on_the_call_log(check) and _names(check, DOOR_FIELD)
618
-
619
-
620
- def _require_the_log(check: dict, state: dict, rows: list[dict]) -> None:
621
- """A check over a call log the engine does not keep would select nothing
622
- (``absent`` passes) and one over a door a log does not record would miss
623
- the key (``neq`` passes): each is ungraded."""
624
- if not _on_the_call_log(check):
625
- return
626
- if _names_the_door(check) and (check["entity"] not in state or any(DOOR_FIELD not in row for row in rows)):
627
- raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
628
- if check["entity"] not in state:
629
- raise GraderError(f"this engine keeps no call log ({check['entity']})")
630
-
631
-
632
- def _unknown_fields(row: dict) -> tuple[str, ...]:
633
- """What a call-log entry cannot say: the request fields it withheld (a
634
- request too large to keep), and ``found`` / ``rows`` where its answer
635
- declared no rows (``found`` null)."""
636
- withheld = tuple(row.get(WITHHELD_FIELD) or ())
637
- return withheld + ROW_FIELDS if row.get("found") is None else withheld
638
-
639
-
640
- def _reads(unknown: tuple[str, ...], path: str) -> bool:
641
- return any(path == field or path.startswith(f"{field}.") for field in unknown)
642
-
643
-
644
- def _possible(check: dict, row: dict, unknown: tuple[str, ...]) -> tuple[bool | None, bool | None]:
645
- """Whether a row is selected and whether it passes the check's predicate,
646
- each None where the row cannot say."""
647
- where = check.get("where") or {}
648
- known = {path: value for path, value in where.items() if not _reads(unknown, path)}
649
- if not _matches_where(row, known):
650
- return False, None
651
- selected = None if len(known) < len(where) else True
652
- return selected, _passes(check, row, unknown)
653
-
654
-
655
- def _passes(check: dict, row: dict, unknown: tuple[str, ...]) -> bool | None:
656
- """Whether a row passes the check's predicate, None where it cannot say. A
657
- withheld list still holds what it lists, so a ``contains`` it already
658
- satisfies passes: more items cannot take it away."""
659
- field = check.get("field")
660
- if field is None:
661
- return None
662
- holds = _predicate(row, field, check["op"], check["value"])
663
- if not _reads(unknown, field):
664
- return holds
665
- partial = isinstance(_lookup(row, field)[1], list)
666
- return True if holds and partial and check["op"] == "contains" else None
667
-
668
-
669
- def _dropped(rows: list[dict]) -> int:
670
- """How many calls a bounded log no longer keeps: the first kept ``seq`` says."""
671
- kept = [row["seq"] for row in rows if isinstance(row.get("seq"), int)]
672
- return min(kept) - 1 if kept else 0
673
-
674
-
675
- def _interval(check: dict) -> tuple[float, float]:
676
- assertion = check["assert"]
677
- if assertion == "exists":
678
- return 1, math.inf
679
- if assertion == "absent":
680
- return 0, 0
681
- bound = check["count"]
682
- return max(bound.get("gte", 0), bound.get("eq", 0)), min(bound.get("lte", math.inf), bound.get("eq", math.inf))
683
-
684
-
685
- def _outcomes(check: dict, possible: list[tuple[bool | None, bool | None]], dropped: int) -> set[bool]:
686
- """Every result the check could have over what the rows cannot say. A call
687
- the log no longer keeps is selected (or, under a ``where``, perhaps) and
688
- passes or fails, unknown."""
689
- gone_sure, gone_maybe = (0, dropped) if check.get("where") else (dropped, 0)
690
- sure = [passes for selected, passes in possible if selected is True] + [None] * min(gone_sure, 1)
691
- maybe = [passes for selected, passes in possible if selected is None] + [None] * min(gone_maybe, 1)
692
- least = len(sure) - min(gone_sure, 1) + gone_sure
693
- most = least + len(maybe) - min(gone_maybe, 1) + gone_maybe
694
- assertion = check["assert"]
695
- if assertion in ("exists", "absent", "count"):
696
- low, high = _interval(check)
697
- can_true, can_false = max(low, least) <= min(high, most), not (low <= least and most <= high)
698
- elif assertion == "all":
699
- can_true = False not in sure and (least > 0 or any(passes is not False for passes in maybe))
700
- can_false = least == 0 or any(passes is not True for passes in sure + maybe)
701
- else:
702
- can_true = any(passes is not False for passes in sure + maybe)
703
- can_false = True not in sure
704
- return {outcome for outcome, holds in ((True, can_true), (False, can_false)) if holds}
705
-
706
-
707
- def evaluate_check(check: dict, state: dict) -> bool:
708
- """One check over a state dump. Over a call log, a row may not say what the
709
- check reads (a withheld request, a ``found`` null, a call the log no longer
710
- keeps): the check grades whenever the rows that can say decide it, and is
711
- ungraded (:class:`GraderError`) only where what they cannot say could
712
- change the result."""
713
- rows = _rows_of(state, check["entity"])
714
- _require_the_log(check, state, rows)
715
- on_log = _on_the_call_log(check)
716
- possible = [_possible(check, row, _unknown_fields(row) if on_log else ()) for row in rows]
717
- outcomes = _outcomes(check, possible, _dropped(rows) if on_log else 0)
718
- if len(outcomes) > 1:
719
- raise GraderError(
720
- "calls this check reads cannot say what it asks (a request too large to keep, an answer that "
721
- "declared no rows, or a call the log no longer keeps), and the rest do not decide it: ungraded"
722
- )
723
- return outcomes.pop()
724
-
725
-
726
- def evaluate_assertions(checks: list[dict], state: dict) -> dict:
727
- """The grade of an assertions grader over a state dump: the weighted share
728
- of checks that passed as ``reward``, each check's 0/1 under its id (or its
729
- position) in ``rewards``. The runner's shape, as ``verify.py`` prints it."""
730
- rewards: dict[str, float] = {}
731
- total = 0.0
732
- passed = 0.0
733
- for i, check in enumerate(checks):
734
- ok = evaluate_check(check, state)
735
- rewards[check_key(check, i)] = 1.0 if ok else 0.0
736
- total += check["weight"]
737
- passed += check["weight"] if ok else 0.0
738
- return {"reward": (passed / total) if total else None, "rewards": rewards}