@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
"""Pure Python over the state dump, so a check cannot read outside its session. Over a log a row may not say
|
|
2
|
+
everything a check asks, so a predicate is three-valued: a check its rows cannot decide fails, with the reason, over
|
|
3
|
+
the journal, and is ungraded over any other log."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import math
|
|
8
|
+
from abc import ABC, abstractmethod
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any, Mapping
|
|
11
|
+
|
|
12
|
+
from .journal import NOT_RECORDED
|
|
13
|
+
from .tasks import (
|
|
14
|
+
CALL_LOG,
|
|
15
|
+
DOOR_FIELD,
|
|
16
|
+
JOURNAL,
|
|
17
|
+
JOURNAL_META,
|
|
18
|
+
ORDER_KEYS,
|
|
19
|
+
ROW_FIELDS,
|
|
20
|
+
WITHHELD_FIELD,
|
|
21
|
+
GraderError,
|
|
22
|
+
_names_the_door,
|
|
23
|
+
check_key,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
RANK = {False: 0, None: 1, True: 2}
|
|
27
|
+
BEFORE = {(True, True): False, (True, False): False, (False, True): None, (False, False): True}
|
|
28
|
+
AFTER = {(True, True): True, (True, False): True, (False, True): None, (False, False): False}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def lookup(row: Any, path: str) -> tuple[bool, Any]:
|
|
32
|
+
current = row
|
|
33
|
+
for part in path.split("."):
|
|
34
|
+
if not isinstance(current, dict) or part not in current:
|
|
35
|
+
return False, None
|
|
36
|
+
current = current[part]
|
|
37
|
+
return True, current
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def same(actual: Any, expected: Any) -> bool:
|
|
41
|
+
"""JSON equality: ``true`` is not ``1``, as it is not in the schema."""
|
|
42
|
+
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
43
|
+
return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
|
|
44
|
+
return actual == expected
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def matches_where(row: dict, where: Mapping[str, Any]) -> bool:
|
|
48
|
+
return all(found and same(actual, expected) for found, actual, expected in _looked_up(row, where))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _looked_up(row: dict, where: Mapping[str, Any]) -> list[tuple[bool, Any, Any]]:
|
|
52
|
+
return [(*lookup(row, path), expected) for path, expected in where.items()]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def comparable(actual: Any, expected: Any) -> bool:
|
|
56
|
+
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
57
|
+
return False
|
|
58
|
+
numbers = (int, float)
|
|
59
|
+
if isinstance(actual, numbers) and isinstance(expected, numbers):
|
|
60
|
+
return True
|
|
61
|
+
return isinstance(actual, str) and isinstance(expected, str)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _contains(actual: Any, expected: Any) -> bool:
|
|
65
|
+
if isinstance(actual, str):
|
|
66
|
+
return isinstance(expected, str) and expected in actual
|
|
67
|
+
if isinstance(actual, list):
|
|
68
|
+
return any(same(item, expected) for item in actual)
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
ORDERED_OPS = {
|
|
73
|
+
"gt": lambda actual, expected: actual > expected,
|
|
74
|
+
"gte": lambda actual, expected: actual >= expected,
|
|
75
|
+
"lt": lambda actual, expected: actual < expected,
|
|
76
|
+
"lte": lambda actual, expected: actual <= expected,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def predicate(row: dict, field: str, op: str, expected: Any) -> bool:
|
|
81
|
+
found, actual = lookup(row, field)
|
|
82
|
+
if op == "neq":
|
|
83
|
+
return not found or not same(actual, expected)
|
|
84
|
+
if not found:
|
|
85
|
+
return False
|
|
86
|
+
if op == "eq":
|
|
87
|
+
return same(actual, expected)
|
|
88
|
+
if op == "in":
|
|
89
|
+
return any(same(actual, option) for option in expected)
|
|
90
|
+
if op == "contains":
|
|
91
|
+
return _contains(actual, expected)
|
|
92
|
+
return comparable(actual, expected) and ORDERED_OPS[op](actual, expected)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def and3(*values: bool | None) -> bool | None:
|
|
96
|
+
"""False wins over unknown (None), unknown over true."""
|
|
97
|
+
return min(values, key=RANK.__getitem__)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _under(path: str, root: str) -> bool:
|
|
101
|
+
return path == root or path.startswith(f"{root}.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _body_state(part: dict) -> str:
|
|
105
|
+
if part.get("truncated") is True:
|
|
106
|
+
return "prefix" if part.get("body") is not None else "unknown"
|
|
107
|
+
if part.get("body") is None and (part.get("bodyBytes") or 0) > 0:
|
|
108
|
+
return "unknown"
|
|
109
|
+
return "whole"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _body_paths(values: dict) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
|
113
|
+
parts = [(f"{side}.body", values.get(side)) for side in ("request", "response")]
|
|
114
|
+
states = [(path, _body_state(part)) for path, part in parts if isinstance(part, dict)]
|
|
115
|
+
unknown = tuple(path for path, state in states if state == "unknown")
|
|
116
|
+
return unknown, tuple(path for path, state in states if state == "prefix")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True)
|
|
120
|
+
class Row:
|
|
121
|
+
values: dict
|
|
122
|
+
unknown: tuple[str, ...] = ()
|
|
123
|
+
partial: tuple[str, ...] = ()
|
|
124
|
+
|
|
125
|
+
@classmethod
|
|
126
|
+
def logged(cls, values: dict) -> Row:
|
|
127
|
+
withheld = tuple(values.get(WITHHELD_FIELD) or ())
|
|
128
|
+
return cls(values, withheld + (ROW_FIELDS if values.get("found") is None else ()))
|
|
129
|
+
|
|
130
|
+
@classmethod
|
|
131
|
+
def journaled(cls, values: dict) -> Row:
|
|
132
|
+
logged = cls.logged(values)
|
|
133
|
+
door = (DOOR_FIELD,) if values.get(DOOR_FIELD) is None else ()
|
|
134
|
+
unknown, partial = _body_paths(values)
|
|
135
|
+
return cls(values, logged.unknown + door + unknown, partial)
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def seq(self) -> int | None:
|
|
139
|
+
seq = self.values.get("seq")
|
|
140
|
+
return seq if isinstance(seq, int) and not isinstance(seq, bool) else None
|
|
141
|
+
|
|
142
|
+
def cannot_say(self, path: str) -> bool:
|
|
143
|
+
if any(_under(path, root) for root in self.unknown):
|
|
144
|
+
return True
|
|
145
|
+
root = next((root for root in self.partial if _under(path, root)), None)
|
|
146
|
+
if root is None:
|
|
147
|
+
return False
|
|
148
|
+
found, value = lookup(self.values, path)
|
|
149
|
+
return path == root or not found or isinstance(value, (dict, list))
|
|
150
|
+
|
|
151
|
+
def growable(self, path: str) -> bool:
|
|
152
|
+
value = lookup(self.values, path)[1]
|
|
153
|
+
return isinstance(value, list) or (isinstance(value, str) and path in self.partial)
|
|
154
|
+
|
|
155
|
+
def selected(self, where: Mapping[str, Any] | None) -> bool | None:
|
|
156
|
+
wanted = where or {}
|
|
157
|
+
known = {path: value for path, value in wanted.items() if not self.cannot_say(path)}
|
|
158
|
+
if not matches_where(self.values, known):
|
|
159
|
+
return False
|
|
160
|
+
return None if len(known) < len(wanted) else True
|
|
161
|
+
|
|
162
|
+
def passes(self, check: dict) -> bool | None:
|
|
163
|
+
"""A withheld list or a cut body still holds what it holds, so a ``contains`` it already satisfies passes."""
|
|
164
|
+
field = check.get("field")
|
|
165
|
+
if field is None:
|
|
166
|
+
return None
|
|
167
|
+
holds = predicate(self.values, field, check["op"], check["value"])
|
|
168
|
+
if not self.cannot_say(field):
|
|
169
|
+
return holds
|
|
170
|
+
return True if holds and check["op"] == "contains" and self.growable(field) else None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@dataclass(frozen=True)
|
|
174
|
+
class Outcome:
|
|
175
|
+
passed: bool
|
|
176
|
+
reason: str | None = None
|
|
177
|
+
matched: tuple[int, ...] = ()
|
|
178
|
+
maybe: tuple[int, ...] = ()
|
|
179
|
+
|
|
180
|
+
def as_dict(self) -> dict:
|
|
181
|
+
out: dict[str, Any] = {"passed": self.passed, "matched": list(self.matched), "maybe": list(self.maybe)}
|
|
182
|
+
if self.reason is not None:
|
|
183
|
+
out["reason"] = self.reason
|
|
184
|
+
return out
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _interval(check: dict) -> tuple[float, float]:
|
|
188
|
+
assertion = check["assert"]
|
|
189
|
+
if assertion == "exists":
|
|
190
|
+
return 1, math.inf
|
|
191
|
+
if assertion == "absent":
|
|
192
|
+
return 0, 0
|
|
193
|
+
bound = check["count"]
|
|
194
|
+
return max(bound.get("gte", 0), bound.get("eq", 0)), min(bound.get("lte", math.inf), bound.get("eq", math.inf))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _counted(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
|
|
198
|
+
low, high = _interval(check)
|
|
199
|
+
can_true = max(low, least) <= min(high, most)
|
|
200
|
+
can_false = not (low <= least and most <= high)
|
|
201
|
+
return _possible_results(can_true, can_false)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _every(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
|
|
205
|
+
can_true = False not in sure and (least > 0 or any(passes is not False for passes in maybe))
|
|
206
|
+
can_false = least == 0 or any(passes is not True for passes in sure + maybe)
|
|
207
|
+
return _possible_results(can_true, can_false)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _some(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
|
|
211
|
+
can_true = any(passes is not False for passes in sure + maybe)
|
|
212
|
+
can_false = True not in sure
|
|
213
|
+
return _possible_results(can_true, can_false)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _possible_results(can_true: bool, can_false: bool) -> set[bool]:
|
|
217
|
+
return {outcome for outcome, holds in ((True, can_true), (False, can_false)) if holds}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
RESULTS = {"exists": _counted, "absent": _counted, "count": _counted, "all": _every, "any": _some}
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@dataclass(frozen=True)
|
|
224
|
+
class Tally:
|
|
225
|
+
possible: list[tuple[bool | None, bool | None]]
|
|
226
|
+
gone_sure: float = 0
|
|
227
|
+
gone_maybe: float = 0
|
|
228
|
+
|
|
229
|
+
def results(self, check: dict) -> set[bool]:
|
|
230
|
+
one_sure, one_maybe = min(self.gone_sure, 1), min(self.gone_maybe, 1)
|
|
231
|
+
sure = [passes for selected, passes in self.possible if selected is True] + [None] * one_sure
|
|
232
|
+
maybe = [passes for selected, passes in self.possible if selected is None] + [None] * one_maybe
|
|
233
|
+
least = len(sure) - one_sure + self.gone_sure
|
|
234
|
+
most = least + len(maybe) - one_maybe + self.gone_maybe
|
|
235
|
+
return RESULTS[check["assert"]](check, sure, maybe, least, most)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class Window:
|
|
239
|
+
"""``before`` keeps every row when none matches; ``after`` keeps none."""
|
|
240
|
+
|
|
241
|
+
def __init__(self, rows: list[Row], incomplete: bool) -> None:
|
|
242
|
+
self.rows = rows
|
|
243
|
+
self.incomplete = incomplete
|
|
244
|
+
|
|
245
|
+
def kept(self, check: dict) -> list[bool | None]:
|
|
246
|
+
kept: list[bool | None] = [True] * len(self.rows)
|
|
247
|
+
for key in ORDER_KEYS:
|
|
248
|
+
side = self._side(key, check[key]) if key in check else kept
|
|
249
|
+
kept = [and3(a, b) for a, b in zip(kept, side)]
|
|
250
|
+
return kept
|
|
251
|
+
|
|
252
|
+
def _side(self, key: str, where: dict) -> list[bool | None]:
|
|
253
|
+
return {"before": self._before, "after": self._after}[key](where)
|
|
254
|
+
|
|
255
|
+
def _before(self, where: dict) -> list[bool | None]:
|
|
256
|
+
sure, maybe, kept = False, self.incomplete, []
|
|
257
|
+
for row in self.rows:
|
|
258
|
+
anchor = row.selected(where)
|
|
259
|
+
sure, maybe = sure or anchor is True, maybe or anchor is None
|
|
260
|
+
kept.append(BEFORE[(sure, maybe)])
|
|
261
|
+
return kept
|
|
262
|
+
|
|
263
|
+
def _after(self, where: dict) -> list[bool | None]:
|
|
264
|
+
sure, maybe, kept = False, self.incomplete, []
|
|
265
|
+
for row in self.rows:
|
|
266
|
+
kept.append(AFTER[(sure, maybe)])
|
|
267
|
+
anchor = row.selected(where)
|
|
268
|
+
sure, maybe = sure or anchor is True, maybe or anchor is None
|
|
269
|
+
return kept
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _rows_of(state: Mapping, entity: str) -> list[dict]:
|
|
273
|
+
held = state.get(entity)
|
|
274
|
+
if isinstance(held, dict):
|
|
275
|
+
return [row for row in held.values() if isinstance(row, dict)]
|
|
276
|
+
if isinstance(held, list):
|
|
277
|
+
return [row for row in held if isinstance(row, dict)]
|
|
278
|
+
return []
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _log_of(entity: str) -> str:
|
|
282
|
+
if entity == JOURNAL or entity.endswith(f".{JOURNAL}"):
|
|
283
|
+
return "journal"
|
|
284
|
+
if entity == CALL_LOG or entity.endswith(f".{CALL_LOG}"):
|
|
285
|
+
return "calls"
|
|
286
|
+
return "entity"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _journal_meta(state: Mapping, entity: str) -> Mapping | None:
|
|
290
|
+
prefix = entity[: entity.rfind(".") + 1]
|
|
291
|
+
meta = state.get(f"{prefix}{JOURNAL_META}")
|
|
292
|
+
return meta if isinstance(meta, Mapping) else None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class Selection(ABC):
|
|
296
|
+
def __init__(self, check: dict, rows: list[Row]) -> None:
|
|
297
|
+
self.check = check
|
|
298
|
+
self.rows = rows
|
|
299
|
+
|
|
300
|
+
@staticmethod
|
|
301
|
+
def of(check: dict, state: Mapping) -> Selection:
|
|
302
|
+
entity = check["entity"]
|
|
303
|
+
meta = _journal_meta(state, entity)
|
|
304
|
+
kind = KINDS[(_log_of(entity), meta is not None)]
|
|
305
|
+
return kind.read(check, state, meta)
|
|
306
|
+
|
|
307
|
+
@classmethod
|
|
308
|
+
@abstractmethod
|
|
309
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection: ...
|
|
310
|
+
|
|
311
|
+
def missing(self) -> tuple[float, float]:
|
|
312
|
+
return 0, 0
|
|
313
|
+
|
|
314
|
+
@abstractmethod
|
|
315
|
+
def undecided(self, outcome: Outcome) -> Outcome: ...
|
|
316
|
+
|
|
317
|
+
def outcome(self) -> Outcome:
|
|
318
|
+
incomplete = any(self.missing())
|
|
319
|
+
kept = Window(self.rows, incomplete).kept(self.check)
|
|
320
|
+
possible = [
|
|
321
|
+
(and3(row.selected(self.check.get("where")), inside), row.passes(self.check))
|
|
322
|
+
for row, inside in zip(self.rows, kept)
|
|
323
|
+
]
|
|
324
|
+
results = Tally(possible, *self.missing()).results(self.check)
|
|
325
|
+
found = Outcome(
|
|
326
|
+
passed=False,
|
|
327
|
+
matched=self._seqs(possible, True),
|
|
328
|
+
maybe=self._seqs(possible, None),
|
|
329
|
+
)
|
|
330
|
+
if len(results) > 1:
|
|
331
|
+
return self.undecided(found)
|
|
332
|
+
return Outcome(results.pop(), None, found.matched, found.maybe)
|
|
333
|
+
|
|
334
|
+
def _seqs(self, possible: list[tuple[bool | None, bool | None]], selected: bool | None) -> tuple[int, ...]:
|
|
335
|
+
return tuple(
|
|
336
|
+
row.seq for row, (chosen, _) in zip(self.rows, possible) if chosen is selected and row.seq is not None
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class EntitySelection(Selection):
|
|
341
|
+
@classmethod
|
|
342
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
343
|
+
return cls(check, [Row(values) for values in _rows_of(state, check["entity"])])
|
|
344
|
+
|
|
345
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
346
|
+
raise GraderError(f"the rows of {self.check['entity']} cannot decide this check: ungraded")
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class CallLogSelection(Selection):
|
|
350
|
+
def __init__(self, check: dict, rows: list[Row], dropped: int) -> None:
|
|
351
|
+
super().__init__(check, rows)
|
|
352
|
+
self.dropped = dropped
|
|
353
|
+
|
|
354
|
+
@classmethod
|
|
355
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
356
|
+
entity = check["entity"]
|
|
357
|
+
rows = _rows_of(state, entity)
|
|
358
|
+
if _names_the_door(check) and (entity not in state or any(DOOR_FIELD not in row for row in rows)):
|
|
359
|
+
raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
|
|
360
|
+
if entity not in state:
|
|
361
|
+
raise GraderError(f"this engine keeps no call log ({entity})")
|
|
362
|
+
kept = [row["seq"] for row in rows if isinstance(row.get("seq"), int)]
|
|
363
|
+
return cls(check, [Row.logged(row) for row in rows], min(kept) - 1 if kept else 0)
|
|
364
|
+
|
|
365
|
+
def missing(self) -> tuple[float, float]:
|
|
366
|
+
narrowed = self.check.get("where") or any(key in self.check for key in ORDER_KEYS)
|
|
367
|
+
return (0, self.dropped) if narrowed else (self.dropped, 0)
|
|
368
|
+
|
|
369
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
370
|
+
raise GraderError(
|
|
371
|
+
"calls this check reads cannot say what it asks (a request too large to keep, an answer that "
|
|
372
|
+
"declared no rows, or a call the log no longer keeps), and the rest do not decide it: ungraded"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
class JournalSelection(Selection):
|
|
377
|
+
def __init__(self, check: dict, rows: list[Row], meta: Mapping) -> None:
|
|
378
|
+
super().__init__(check, rows)
|
|
379
|
+
self.meta = meta
|
|
380
|
+
|
|
381
|
+
@classmethod
|
|
382
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
383
|
+
assert meta is not None
|
|
384
|
+
return cls(check, [Row.journaled(row) for row in _rows_of(state, check["entity"])], meta)
|
|
385
|
+
|
|
386
|
+
def missing(self) -> tuple[float, float]:
|
|
387
|
+
return (0, 0) if self.meta.get("complete") is True else (0, math.inf)
|
|
388
|
+
|
|
389
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
390
|
+
return Outcome(False, self._reason(outcome), outcome.matched, outcome.maybe)
|
|
391
|
+
|
|
392
|
+
def _reason(self, outcome: Outcome) -> str:
|
|
393
|
+
if self.meta.get("complete") is not True:
|
|
394
|
+
why = self.meta.get("reason") or "the platform could not prove it holds every request"
|
|
395
|
+
return f"journal incomplete ({why}): the {len(self.rows)} requests it holds do not decide this check"
|
|
396
|
+
which = f"requests {', '.join(str(seq) for seq in outcome.maybe)}" if outcome.maybe else "some requests"
|
|
397
|
+
return (
|
|
398
|
+
f"{which} cannot say what this check reads (a body cut at the capture bound, or not kept), "
|
|
399
|
+
"and the rest do not decide it"
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class UnrecordedSelection(Selection):
|
|
404
|
+
@classmethod
|
|
405
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
406
|
+
raise GraderError(NOT_RECORDED)
|
|
407
|
+
|
|
408
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
409
|
+
raise GraderError(NOT_RECORDED)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
KINDS: dict[tuple[str, bool], type[Selection]] = {
|
|
413
|
+
("journal", True): JournalSelection,
|
|
414
|
+
("journal", False): UnrecordedSelection,
|
|
415
|
+
("calls", True): JournalSelection,
|
|
416
|
+
("calls", False): CallLogSelection,
|
|
417
|
+
("entity", True): EntitySelection,
|
|
418
|
+
("entity", False): EntitySelection,
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
@dataclass(frozen=True)
|
|
423
|
+
class AssertionsGrade:
|
|
424
|
+
reward: float | None
|
|
425
|
+
rewards: dict[str, float]
|
|
426
|
+
reasons: dict[str, str]
|
|
427
|
+
outcomes: dict[str, Outcome]
|
|
428
|
+
|
|
429
|
+
def as_dict(self) -> dict:
|
|
430
|
+
out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
|
|
431
|
+
if self.reasons:
|
|
432
|
+
out["reasons"] = dict(self.reasons)
|
|
433
|
+
return out
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
class Assertions:
|
|
437
|
+
def __init__(self, checks: list[dict]) -> None:
|
|
438
|
+
self.checks = checks
|
|
439
|
+
|
|
440
|
+
def grade(self, state: Mapping) -> AssertionsGrade:
|
|
441
|
+
outcomes = {check_key(check, i): Selection.of(check, state).outcome() for i, check in enumerate(self.checks)}
|
|
442
|
+
weights = {check_key(check, i): check["weight"] for i, check in enumerate(self.checks)}
|
|
443
|
+
total = sum(weights.values())
|
|
444
|
+
passed = sum(weights[key] for key, outcome in outcomes.items() if outcome.passed)
|
|
445
|
+
return AssertionsGrade(
|
|
446
|
+
reward=(passed / total) if total else None,
|
|
447
|
+
rewards={key: 1.0 if outcome.passed else 0.0 for key, outcome in outcomes.items()},
|
|
448
|
+
reasons={key: outcome.reason for key, outcome in outcomes.items() if outcome.reason is not None},
|
|
449
|
+
outcomes=outcomes,
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def evaluate_check(check: dict, state: Mapping) -> bool:
|
|
454
|
+
return Selection.of(check, state).outcome().passed
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def evaluate_assertions(checks: list[dict], state: Mapping) -> dict:
|
|
458
|
+
return Assertions(checks).grade(state).as_dict()
|
|
@@ -28,9 +28,9 @@ from typing import IO, Any, Iterable, Iterator, NoReturn
|
|
|
28
28
|
from .cast import TEXT_CASTER, EntityCasters
|
|
29
29
|
from .compiler import Contract, payload_value
|
|
30
30
|
from .csv_rows import csv_delimiter, is_delimited_name, iter_csv_records
|
|
31
|
+
from .database import WorldDatabase
|
|
31
32
|
from .primary_key import PrimaryKey
|
|
32
33
|
from .redaction import RowRedactor
|
|
33
|
-
from .database import WorldDatabase
|
|
34
34
|
from .validation import ContractError, fail, validate
|
|
35
35
|
|
|
36
36
|
CHUNK_BYTES = 64 * 1024 * 1024
|
|
@@ -283,10 +283,10 @@ def iter_csv_rows(
|
|
|
283
283
|
if seen is not None:
|
|
284
284
|
seen.setdefault(entity, 0)
|
|
285
285
|
caster = (casters(entity) if casters else None) or TEXT_CASTER
|
|
286
|
-
for
|
|
286
|
+
for _number, record in iter_csv_records(stream, where, delimiter):
|
|
287
287
|
if seen is not None:
|
|
288
288
|
seen[entity] += 1
|
|
289
|
-
yield entity, caster.row(record
|
|
289
|
+
yield entity, caster.row(record)
|
|
290
290
|
|
|
291
291
|
|
|
292
292
|
def iter_rows(
|