@withgateway/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2880 -2032
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2982 -2135
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +180 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +172 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1309 -540
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1287 -532
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +177 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +169 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +69 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +69 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +249 -120
- package/dist/worlds/index.d.ts +249 -120
- package/dist/worlds/index.js +1240 -484
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1231 -476
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -514
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +19 -11
- package/skills/connector-schema-authoring/connector-worlds.md +13 -4
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +14 -14
- package/skills/worlds-getting-started/SKILL.md +12 -12
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
A custom tool in ``tools/handlers.py`` reaches the world's rows through
|
|
4
4
|
``from gateway_env import store``. That object is a facade over the schema
|
|
5
|
-
store (:class:`~gatewaysdk.environment.schema.
|
|
5
|
+
store (:class:`~gatewaysdk.environment.schema.database.WorldDatabase`) exposing
|
|
6
6
|
exactly the methods named here — nothing else, so a handler can never reach
|
|
7
7
|
the store's own transaction, snapshot or raw-SQL machinery.
|
|
8
8
|
|
|
@@ -10,7 +10,7 @@ This module is the single source of the contract: the runtime facade admits
|
|
|
10
10
|
these names, ``schema check`` flags any other ``store.<name>`` statically,
|
|
11
11
|
``schema init`` writes them into the scaffolded handler docstring, and a test
|
|
12
12
|
holds the skills and CLI help to the same list. ``describe_store`` reads the
|
|
13
|
-
signature off the
|
|
13
|
+
signature off the base class every engine extends, so the printed contract cannot drift from it.
|
|
14
14
|
"""
|
|
15
15
|
|
|
16
16
|
from __future__ import annotations
|
|
@@ -18,14 +18,10 @@ from __future__ import annotations
|
|
|
18
18
|
import inspect
|
|
19
19
|
from dataclasses import dataclass
|
|
20
20
|
|
|
21
|
-
from .
|
|
21
|
+
from .database import WorldDatabase
|
|
22
22
|
|
|
23
|
-
#: Every public method of the store class by name, read once with `inspect`
|
|
24
|
-
#: so a contract entry resolves through an explicit table, not attribute lookup.
|
|
25
23
|
_STORE_MEMBERS: dict[str, object] = {
|
|
26
|
-
name: member
|
|
27
|
-
for name, member in inspect.getmembers(SQLiteStore, inspect.isfunction)
|
|
28
|
-
if not name.startswith("_")
|
|
24
|
+
name: member for name, member in inspect.getmembers(WorldDatabase, inspect.isfunction) if not name.startswith("_")
|
|
29
25
|
}
|
|
30
26
|
|
|
31
27
|
|
|
@@ -38,7 +34,7 @@ class StoreMethod:
|
|
|
38
34
|
|
|
39
35
|
@property
|
|
40
36
|
def signature(self) -> str:
|
|
41
|
-
"""``name(entity, key) -> dict | None`` — read off :class:`
|
|
37
|
+
"""``name(entity, key) -> dict | None`` — read off :class:`WorldDatabase`."""
|
|
42
38
|
sig = inspect.signature(_STORE_MEMBERS[self.name])
|
|
43
39
|
params = [p for p in sig.parameters.values() if p.name != "self"]
|
|
44
40
|
# The store is written under `from __future__ import annotations`, so
|
|
@@ -51,8 +47,8 @@ class StoreMethod:
|
|
|
51
47
|
|
|
52
48
|
|
|
53
49
|
#: The contract, in the order the docs list it. A name here MUST be a public
|
|
54
|
-
#: method of ``
|
|
55
|
-
#: is not a handler's to call.
|
|
50
|
+
#: method of ``WorldDatabase`` (tested); a method of the database that is not
|
|
51
|
+
#: here is not a handler's to call.
|
|
56
52
|
STORE_METHODS: tuple[StoreMethod, ...] = (
|
|
57
53
|
StoreMethod(
|
|
58
54
|
"get",
|
|
@@ -71,9 +67,7 @@ STORE_METHODS: tuple[StoreMethod, ...] = (
|
|
|
71
67
|
"update",
|
|
72
68
|
"Merge `patch` into the row at `key` and validate the result; returns the row.",
|
|
73
69
|
),
|
|
74
|
-
StoreMethod(
|
|
75
|
-
"delete", "Remove the row at `key`; a missing row is a contract error."
|
|
76
|
-
),
|
|
70
|
+
StoreMethod("delete", "Remove the row at `key`; a missing row is a contract error."),
|
|
77
71
|
StoreMethod(
|
|
78
72
|
"apply_batch",
|
|
79
73
|
"Apply one ingest batch {id, target, mapping, coverage, mutations} atomically; an exact retry is a no-op.",
|
|
@@ -21,11 +21,13 @@ Keep the two in step by hand; the limits and the check grammar are the same.
|
|
|
21
21
|
from __future__ import annotations
|
|
22
22
|
|
|
23
23
|
import json
|
|
24
|
+
import math
|
|
24
25
|
import re
|
|
25
26
|
from dataclasses import dataclass
|
|
26
27
|
from pathlib import Path
|
|
27
28
|
from typing import Any, Iterable
|
|
28
29
|
|
|
30
|
+
from ..reward_keys import RewardKeys
|
|
29
31
|
from .validation import fail
|
|
30
32
|
|
|
31
33
|
# The caps, as the platform's WORLD_TASK_LIMITS.
|
|
@@ -114,9 +116,7 @@ class TaskSpec:
|
|
|
114
116
|
own = f"{self.name}."
|
|
115
117
|
for verifier in self.verifiers:
|
|
116
118
|
named.extend(
|
|
117
|
-
check["entity"][len(own) :]
|
|
118
|
-
for check in verifier.get("checks", ())
|
|
119
|
-
if check["entity"].startswith(own)
|
|
119
|
+
check["entity"][len(own) :] for check in verifier.get("checks", ()) if check["entity"].startswith(own)
|
|
120
120
|
)
|
|
121
121
|
return list(dict.fromkeys(named))
|
|
122
122
|
|
|
@@ -128,6 +128,8 @@ class TaskSpec:
|
|
|
128
128
|
RESERVED_STATE_KEYS = frozenset({"calls", "snapshot"})
|
|
129
129
|
CALL_LOG = "calls"
|
|
130
130
|
DOOR_FIELD = "via"
|
|
131
|
+
WITHHELD_FIELD = "withheld"
|
|
132
|
+
ROW_FIELDS = ("found", "rows")
|
|
131
133
|
|
|
132
134
|
|
|
133
135
|
class GraderError(Exception):
|
|
@@ -139,8 +141,11 @@ def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
|
|
|
139
141
|
A grader's check on a reserved state key is not one of them."""
|
|
140
142
|
known = set(defined)
|
|
141
143
|
seeded = set((spec.seed or {}).get("rows", {})) if isinstance(spec.seed, dict) else set()
|
|
142
|
-
return [
|
|
143
|
-
|
|
144
|
+
return [
|
|
145
|
+
entity
|
|
146
|
+
for entity in spec.entities()
|
|
147
|
+
if entity not in known and not (entity in RESERVED_STATE_KEYS and entity not in seeded)
|
|
148
|
+
]
|
|
144
149
|
|
|
145
150
|
|
|
146
151
|
def parse_task_spec(value: Any) -> TaskSpec:
|
|
@@ -163,7 +168,11 @@ def parse_task_spec(value: Any) -> TaskSpec:
|
|
|
163
168
|
tools = _parse_tools(value.get("tools"))
|
|
164
169
|
verifiers = _parse_verifiers(value.get("verifiers"), name)
|
|
165
170
|
if verifiers and grader is not None and grader["kind"] == "rubric":
|
|
166
|
-
fail(
|
|
171
|
+
fail(
|
|
172
|
+
"task",
|
|
173
|
+
"a task with verifiers grades by assertions, python or nothing: a rubric never reaches the host's grade",
|
|
174
|
+
"/task/grader/kind",
|
|
175
|
+
)
|
|
167
176
|
metadata = value.get("metadata")
|
|
168
177
|
if metadata is None:
|
|
169
178
|
metadata = {}
|
|
@@ -227,12 +236,23 @@ def _parse_verifier(row: Any, at: str, task_name: str | None) -> dict:
|
|
|
227
236
|
if not isinstance(name, str) or not _VERIFIER_NAME.match(name) or len(name) > NAME_CHARS:
|
|
228
237
|
fail("task", "a verifier name is letters, digits, _ . -", f"{at}/name")
|
|
229
238
|
scope = row.get("scope")
|
|
230
|
-
if
|
|
239
|
+
if (
|
|
240
|
+
not isinstance(scope, list)
|
|
241
|
+
or len(scope) < 2
|
|
242
|
+
or any(not isinstance(a, str) or not _NAME.match(a) for a in scope)
|
|
243
|
+
):
|
|
231
244
|
fail("task", "a verifier's scope names two or more world aliases", f"{at}/scope")
|
|
232
245
|
if len(set(scope)) != len(scope):
|
|
233
246
|
fail("task", "a verifier's scope lists each alias once", f"{at}/scope")
|
|
247
|
+
reserved = [alias for alias in scope if RewardKeys.reserved(alias)]
|
|
248
|
+
if reserved:
|
|
249
|
+
fail("task", RewardKeys.refusal(reserved[0]), f"{at}/scope")
|
|
234
250
|
if scope[0] != task_name:
|
|
235
|
-
fail(
|
|
251
|
+
fail(
|
|
252
|
+
"task",
|
|
253
|
+
f"verifier '{name}' runs on the session of its first scope alias, so the task's name must be '{scope[0]}'",
|
|
254
|
+
f"{at}/scope",
|
|
255
|
+
)
|
|
236
256
|
grader = _parse_grader({k: v for k, v in row.items() if k not in ("name", "scope")}, at)
|
|
237
257
|
if grader is None or grader["kind"] == "rubric":
|
|
238
258
|
fail("task", "a verifier grades by assertions or python (a rubric judge reads one world)", f"{at}/kind")
|
|
@@ -266,21 +286,15 @@ def _parse_tools(tools: Any) -> tuple[str, ...] | None:
|
|
|
266
286
|
return tuple(tools)
|
|
267
287
|
|
|
268
288
|
|
|
269
|
-
def unknown_task_tools(spec: TaskSpec, served: Iterable[str]) -> list[str]:
|
|
270
|
-
"""The tools ``spec`` allows that the world does not serve. A name the
|
|
271
|
-
world never had would make the allow-list smaller than the author meant
|
|
272
|
-
without anyone noticing, so it is refused at open."""
|
|
273
|
-
if spec.tools is None:
|
|
274
|
-
return []
|
|
275
|
-
known = set(served)
|
|
276
|
-
return [tool for tool in spec.tools if tool not in known]
|
|
277
|
-
|
|
278
|
-
|
|
279
289
|
def _parse_seed(seed: Any) -> dict | None:
|
|
280
290
|
if seed is None:
|
|
281
291
|
return None
|
|
282
292
|
if not isinstance(seed, dict) or set(seed) - {"mode", "rows", "redact"}:
|
|
283
|
-
fail(
|
|
293
|
+
fail(
|
|
294
|
+
"task",
|
|
295
|
+
"seed is {mode?: append|replace, redact?: off|apply|refuse, rows: {entity: [row, ...]}}",
|
|
296
|
+
"/task/seed",
|
|
297
|
+
)
|
|
284
298
|
mode = seed.get("mode", "append")
|
|
285
299
|
if mode not in SEED_MODES:
|
|
286
300
|
fail("task", f"seed mode must be one of {', '.join(SEED_MODES)}", "/task/seed/mode")
|
|
@@ -488,11 +502,6 @@ def _parse_check(check: Any, at: str) -> dict:
|
|
|
488
502
|
return out
|
|
489
503
|
|
|
490
504
|
|
|
491
|
-
# --------------------------------------------------------------------------- #
|
|
492
|
-
# On disk
|
|
493
|
-
# --------------------------------------------------------------------------- #
|
|
494
|
-
|
|
495
|
-
|
|
496
505
|
def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
|
|
497
506
|
"""The task under the session: ``task.json`` (the whole spec),
|
|
498
507
|
``instruction.md``, and for a python grader ``checks/verify.py`` where
|
|
@@ -522,11 +531,6 @@ def read_task_dir(session_dir: Path) -> TaskSpec | None:
|
|
|
522
531
|
return parse_task_spec(json.loads(path.read_text(encoding="utf-8")))
|
|
523
532
|
|
|
524
533
|
|
|
525
|
-
# --------------------------------------------------------------------------- #
|
|
526
|
-
# The assertions grader
|
|
527
|
-
# --------------------------------------------------------------------------- #
|
|
528
|
-
|
|
529
|
-
|
|
530
534
|
def _lookup(row: Any, path: str) -> tuple[bool, Any]:
|
|
531
535
|
"""``(found, value)`` of a dotted path into a row."""
|
|
532
536
|
current = row
|
|
@@ -586,7 +590,7 @@ def _predicate(row: dict, field: str, op: str, expected: Any) -> bool:
|
|
|
586
590
|
return actual >= expected
|
|
587
591
|
if op == "lt":
|
|
588
592
|
return actual < expected
|
|
589
|
-
return actual <= expected
|
|
593
|
+
return actual <= expected
|
|
590
594
|
|
|
591
595
|
|
|
592
596
|
def _rows_of(state: dict, entity: str) -> list[dict]:
|
|
@@ -600,42 +604,123 @@ def _rows_of(state: dict, entity: str) -> list[dict]:
|
|
|
600
604
|
return []
|
|
601
605
|
|
|
602
606
|
|
|
603
|
-
def
|
|
607
|
+
def _on_the_call_log(check: dict) -> bool:
|
|
604
608
|
entity = check["entity"]
|
|
605
|
-
|
|
606
|
-
|
|
609
|
+
return entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def _names(check: dict, field: str) -> bool:
|
|
613
|
+
return check.get("field") == field or field in (check.get("where") or {})
|
|
607
614
|
|
|
608
615
|
|
|
609
|
-
def
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
616
|
+
def _names_the_door(check: dict) -> bool:
|
|
617
|
+
return _on_the_call_log(check) and _names(check, DOOR_FIELD)
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _require_the_log(check: dict, state: dict, rows: list[dict]) -> None:
|
|
621
|
+
"""A check over a call log the engine does not keep would select nothing
|
|
622
|
+
(``absent`` passes) and one over a door a log does not record would miss
|
|
623
|
+
the key (``neq`` passes): each is ungraded."""
|
|
624
|
+
if not _on_the_call_log(check):
|
|
614
625
|
return
|
|
615
|
-
if check["entity"] not in state or any(DOOR_FIELD not in row for row in rows):
|
|
626
|
+
if _names_the_door(check) and (check["entity"] not in state or any(DOOR_FIELD not in row for row in rows)):
|
|
616
627
|
raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
|
|
628
|
+
if check["entity"] not in state:
|
|
629
|
+
raise GraderError(f"this engine keeps no call log ({check['entity']})")
|
|
617
630
|
|
|
618
631
|
|
|
619
|
-
def
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
632
|
+
def _unknown_fields(row: dict) -> tuple[str, ...]:
|
|
633
|
+
"""What a call-log entry cannot say: the request fields it withheld (a
|
|
634
|
+
request too large to keep), and ``found`` / ``rows`` where its answer
|
|
635
|
+
declared no rows (``found`` null)."""
|
|
636
|
+
withheld = tuple(row.get(WITHHELD_FIELD) or ())
|
|
637
|
+
return withheld + ROW_FIELDS if row.get("found") is None else withheld
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def _reads(unknown: tuple[str, ...], path: str) -> bool:
|
|
641
|
+
return any(path == field or path.startswith(f"{field}.") for field in unknown)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _possible(check: dict, row: dict, unknown: tuple[str, ...]) -> tuple[bool | None, bool | None]:
|
|
645
|
+
"""Whether a row is selected and whether it passes the check's predicate,
|
|
646
|
+
each None where the row cannot say."""
|
|
647
|
+
where = check.get("where") or {}
|
|
648
|
+
known = {path: value for path, value in where.items() if not _reads(unknown, path)}
|
|
649
|
+
if not _matches_where(row, known):
|
|
650
|
+
return False, None
|
|
651
|
+
selected = None if len(known) < len(where) else True
|
|
652
|
+
return selected, _passes(check, row, unknown)
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _passes(check: dict, row: dict, unknown: tuple[str, ...]) -> bool | None:
|
|
656
|
+
"""Whether a row passes the check's predicate, None where it cannot say. A
|
|
657
|
+
withheld list still holds what it lists, so a ``contains`` it already
|
|
658
|
+
satisfies passes: more items cannot take it away."""
|
|
659
|
+
field = check.get("field")
|
|
660
|
+
if field is None:
|
|
661
|
+
return None
|
|
662
|
+
holds = _predicate(row, field, check["op"], check["value"])
|
|
663
|
+
if not _reads(unknown, field):
|
|
664
|
+
return holds
|
|
665
|
+
partial = isinstance(_lookup(row, field)[1], list)
|
|
666
|
+
return True if holds and partial and check["op"] == "contains" else None
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _dropped(rows: list[dict]) -> int:
|
|
670
|
+
"""How many calls a bounded log no longer keeps: the first kept ``seq`` says."""
|
|
671
|
+
kept = [row["seq"] for row in rows if isinstance(row.get("seq"), int)]
|
|
672
|
+
return min(kept) - 1 if kept else 0
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _interval(check: dict) -> tuple[float, float]:
|
|
623
676
|
assertion = check["assert"]
|
|
624
677
|
if assertion == "exists":
|
|
625
|
-
return
|
|
678
|
+
return 1, math.inf
|
|
626
679
|
if assertion == "absent":
|
|
627
|
-
return
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
680
|
+
return 0, 0
|
|
681
|
+
bound = check["count"]
|
|
682
|
+
return max(bound.get("gte", 0), bound.get("eq", 0)), min(bound.get("lte", math.inf), bound.get("eq", math.inf))
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def _outcomes(check: dict, possible: list[tuple[bool | None, bool | None]], dropped: int) -> set[bool]:
|
|
686
|
+
"""Every result the check could have over what the rows cannot say. A call
|
|
687
|
+
the log no longer keeps is selected (or, under a ``where``, perhaps) and
|
|
688
|
+
passes or fails, unknown."""
|
|
689
|
+
gone_sure, gone_maybe = (0, dropped) if check.get("where") else (dropped, 0)
|
|
690
|
+
sure = [passes for selected, passes in possible if selected is True] + [None] * min(gone_sure, 1)
|
|
691
|
+
maybe = [passes for selected, passes in possible if selected is None] + [None] * min(gone_maybe, 1)
|
|
692
|
+
least = len(sure) - min(gone_sure, 1) + gone_sure
|
|
693
|
+
most = least + len(maybe) - min(gone_maybe, 1) + gone_maybe
|
|
694
|
+
assertion = check["assert"]
|
|
695
|
+
if assertion in ("exists", "absent", "count"):
|
|
696
|
+
low, high = _interval(check)
|
|
697
|
+
can_true, can_false = max(low, least) <= min(high, most), not (low <= least and most <= high)
|
|
698
|
+
elif assertion == "all":
|
|
699
|
+
can_true = False not in sure and (least > 0 or any(passes is not False for passes in maybe))
|
|
700
|
+
can_false = least == 0 or any(passes is not True for passes in sure + maybe)
|
|
701
|
+
else:
|
|
702
|
+
can_true = any(passes is not False for passes in sure + maybe)
|
|
703
|
+
can_false = True not in sure
|
|
704
|
+
return {outcome for outcome, holds in ((True, can_true), (False, can_false)) if holds}
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def evaluate_check(check: dict, state: dict) -> bool:
|
|
708
|
+
"""One check over a state dump. Over a call log, a row may not say what the
|
|
709
|
+
check reads (a withheld request, a ``found`` null, a call the log no longer
|
|
710
|
+
keeps): the check grades whenever the rows that can say decide it, and is
|
|
711
|
+
ungraded (:class:`GraderError`) only where what they cannot say could
|
|
712
|
+
change the result."""
|
|
713
|
+
rows = _rows_of(state, check["entity"])
|
|
714
|
+
_require_the_log(check, state, rows)
|
|
715
|
+
on_log = _on_the_call_log(check)
|
|
716
|
+
possible = [_possible(check, row, _unknown_fields(row) if on_log else ()) for row in rows]
|
|
717
|
+
outcomes = _outcomes(check, possible, _dropped(rows) if on_log else 0)
|
|
718
|
+
if len(outcomes) > 1:
|
|
719
|
+
raise GraderError(
|
|
720
|
+
"calls this check reads cannot say what it asks (a request too large to keep, an answer that "
|
|
721
|
+
"declared no rows, or a call the log no longer keeps), and the rest do not decide it: ungraded"
|
|
634
722
|
)
|
|
635
|
-
|
|
636
|
-
if assertion == "all":
|
|
637
|
-
return bool(selected) and all(tests)
|
|
638
|
-
return any(tests) # any
|
|
723
|
+
return outcomes.pop()
|
|
639
724
|
|
|
640
725
|
|
|
641
726
|
def evaluate_assertions(checks: list[dict], state: dict) -> dict:
|