@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,458 @@
1
+ """Pure Python over the state dump, so a check cannot read outside its session. Over a log a row may not say
2
+ everything a check asks, so a predicate is three-valued: a check its rows cannot decide fails, with the reason, over
3
+ the journal, and is ungraded over any other log."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import math
8
+ from abc import ABC, abstractmethod
9
+ from dataclasses import dataclass
10
+ from typing import Any, Mapping
11
+
12
+ from .journal import NOT_RECORDED
13
+ from .tasks import (
14
+ CALL_LOG,
15
+ DOOR_FIELD,
16
+ JOURNAL,
17
+ JOURNAL_META,
18
+ ORDER_KEYS,
19
+ ROW_FIELDS,
20
+ WITHHELD_FIELD,
21
+ GraderError,
22
+ _names_the_door,
23
+ check_key,
24
+ )
25
+
26
+ RANK = {False: 0, None: 1, True: 2}
27
+ BEFORE = {(True, True): False, (True, False): False, (False, True): None, (False, False): True}
28
+ AFTER = {(True, True): True, (True, False): True, (False, True): None, (False, False): False}
29
+
30
+
31
+ def lookup(row: Any, path: str) -> tuple[bool, Any]:
32
+ current = row
33
+ for part in path.split("."):
34
+ if not isinstance(current, dict) or part not in current:
35
+ return False, None
36
+ current = current[part]
37
+ return True, current
38
+
39
+
40
+ def same(actual: Any, expected: Any) -> bool:
41
+ """JSON equality: ``true`` is not ``1``, as it is not in the schema."""
42
+ if isinstance(actual, bool) or isinstance(expected, bool):
43
+ return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
44
+ return actual == expected
45
+
46
+
47
+ def matches_where(row: dict, where: Mapping[str, Any]) -> bool:
48
+ return all(found and same(actual, expected) for found, actual, expected in _looked_up(row, where))
49
+
50
+
51
+ def _looked_up(row: dict, where: Mapping[str, Any]) -> list[tuple[bool, Any, Any]]:
52
+ return [(*lookup(row, path), expected) for path, expected in where.items()]
53
+
54
+
55
+ def comparable(actual: Any, expected: Any) -> bool:
56
+ if isinstance(actual, bool) or isinstance(expected, bool):
57
+ return False
58
+ numbers = (int, float)
59
+ if isinstance(actual, numbers) and isinstance(expected, numbers):
60
+ return True
61
+ return isinstance(actual, str) and isinstance(expected, str)
62
+
63
+
64
+ def _contains(actual: Any, expected: Any) -> bool:
65
+ if isinstance(actual, str):
66
+ return isinstance(expected, str) and expected in actual
67
+ if isinstance(actual, list):
68
+ return any(same(item, expected) for item in actual)
69
+ return False
70
+
71
+
72
+ ORDERED_OPS = {
73
+ "gt": lambda actual, expected: actual > expected,
74
+ "gte": lambda actual, expected: actual >= expected,
75
+ "lt": lambda actual, expected: actual < expected,
76
+ "lte": lambda actual, expected: actual <= expected,
77
+ }
78
+
79
+
80
+ def predicate(row: dict, field: str, op: str, expected: Any) -> bool:
81
+ found, actual = lookup(row, field)
82
+ if op == "neq":
83
+ return not found or not same(actual, expected)
84
+ if not found:
85
+ return False
86
+ if op == "eq":
87
+ return same(actual, expected)
88
+ if op == "in":
89
+ return any(same(actual, option) for option in expected)
90
+ if op == "contains":
91
+ return _contains(actual, expected)
92
+ return comparable(actual, expected) and ORDERED_OPS[op](actual, expected)
93
+
94
+
95
+ def and3(*values: bool | None) -> bool | None:
96
+ """False wins over unknown (None), unknown over true."""
97
+ return min(values, key=RANK.__getitem__)
98
+
99
+
100
+ def _under(path: str, root: str) -> bool:
101
+ return path == root or path.startswith(f"{root}.")
102
+
103
+
104
+ def _body_state(part: dict) -> str:
105
+ if part.get("truncated") is True:
106
+ return "prefix" if part.get("body") is not None else "unknown"
107
+ if part.get("body") is None and (part.get("bodyBytes") or 0) > 0:
108
+ return "unknown"
109
+ return "whole"
110
+
111
+
112
+ def _body_paths(values: dict) -> tuple[tuple[str, ...], tuple[str, ...]]:
113
+ parts = [(f"{side}.body", values.get(side)) for side in ("request", "response")]
114
+ states = [(path, _body_state(part)) for path, part in parts if isinstance(part, dict)]
115
+ unknown = tuple(path for path, state in states if state == "unknown")
116
+ return unknown, tuple(path for path, state in states if state == "prefix")
117
+
118
+
119
+ @dataclass(frozen=True)
120
+ class Row:
121
+ values: dict
122
+ unknown: tuple[str, ...] = ()
123
+ partial: tuple[str, ...] = ()
124
+
125
+ @classmethod
126
+ def logged(cls, values: dict) -> Row:
127
+ withheld = tuple(values.get(WITHHELD_FIELD) or ())
128
+ return cls(values, withheld + (ROW_FIELDS if values.get("found") is None else ()))
129
+
130
+ @classmethod
131
+ def journaled(cls, values: dict) -> Row:
132
+ logged = cls.logged(values)
133
+ door = (DOOR_FIELD,) if values.get(DOOR_FIELD) is None else ()
134
+ unknown, partial = _body_paths(values)
135
+ return cls(values, logged.unknown + door + unknown, partial)
136
+
137
+ @property
138
+ def seq(self) -> int | None:
139
+ seq = self.values.get("seq")
140
+ return seq if isinstance(seq, int) and not isinstance(seq, bool) else None
141
+
142
+ def cannot_say(self, path: str) -> bool:
143
+ if any(_under(path, root) for root in self.unknown):
144
+ return True
145
+ root = next((root for root in self.partial if _under(path, root)), None)
146
+ if root is None:
147
+ return False
148
+ found, value = lookup(self.values, path)
149
+ return path == root or not found or isinstance(value, (dict, list))
150
+
151
+ def growable(self, path: str) -> bool:
152
+ value = lookup(self.values, path)[1]
153
+ return isinstance(value, list) or (isinstance(value, str) and path in self.partial)
154
+
155
+ def selected(self, where: Mapping[str, Any] | None) -> bool | None:
156
+ wanted = where or {}
157
+ known = {path: value for path, value in wanted.items() if not self.cannot_say(path)}
158
+ if not matches_where(self.values, known):
159
+ return False
160
+ return None if len(known) < len(wanted) else True
161
+
162
+ def passes(self, check: dict) -> bool | None:
163
+ """A withheld list or a cut body still holds what it holds, so a ``contains`` it already satisfies passes."""
164
+ field = check.get("field")
165
+ if field is None:
166
+ return None
167
+ holds = predicate(self.values, field, check["op"], check["value"])
168
+ if not self.cannot_say(field):
169
+ return holds
170
+ return True if holds and check["op"] == "contains" and self.growable(field) else None
171
+
172
+
173
+ @dataclass(frozen=True)
174
+ class Outcome:
175
+ passed: bool
176
+ reason: str | None = None
177
+ matched: tuple[int, ...] = ()
178
+ maybe: tuple[int, ...] = ()
179
+
180
+ def as_dict(self) -> dict:
181
+ out: dict[str, Any] = {"passed": self.passed, "matched": list(self.matched), "maybe": list(self.maybe)}
182
+ if self.reason is not None:
183
+ out["reason"] = self.reason
184
+ return out
185
+
186
+
187
+ def _interval(check: dict) -> tuple[float, float]:
188
+ assertion = check["assert"]
189
+ if assertion == "exists":
190
+ return 1, math.inf
191
+ if assertion == "absent":
192
+ return 0, 0
193
+ bound = check["count"]
194
+ return max(bound.get("gte", 0), bound.get("eq", 0)), min(bound.get("lte", math.inf), bound.get("eq", math.inf))
195
+
196
+
197
+ def _counted(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
198
+ low, high = _interval(check)
199
+ can_true = max(low, least) <= min(high, most)
200
+ can_false = not (low <= least and most <= high)
201
+ return _possible_results(can_true, can_false)
202
+
203
+
204
+ def _every(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
205
+ can_true = False not in sure and (least > 0 or any(passes is not False for passes in maybe))
206
+ can_false = least == 0 or any(passes is not True for passes in sure + maybe)
207
+ return _possible_results(can_true, can_false)
208
+
209
+
210
+ def _some(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
211
+ can_true = any(passes is not False for passes in sure + maybe)
212
+ can_false = True not in sure
213
+ return _possible_results(can_true, can_false)
214
+
215
+
216
+ def _possible_results(can_true: bool, can_false: bool) -> set[bool]:
217
+ return {outcome for outcome, holds in ((True, can_true), (False, can_false)) if holds}
218
+
219
+
220
+ RESULTS = {"exists": _counted, "absent": _counted, "count": _counted, "all": _every, "any": _some}
221
+
222
+
223
+ @dataclass(frozen=True)
224
+ class Tally:
225
+ possible: list[tuple[bool | None, bool | None]]
226
+ gone_sure: float = 0
227
+ gone_maybe: float = 0
228
+
229
+ def results(self, check: dict) -> set[bool]:
230
+ one_sure, one_maybe = min(self.gone_sure, 1), min(self.gone_maybe, 1)
231
+ sure = [passes for selected, passes in self.possible if selected is True] + [None] * one_sure
232
+ maybe = [passes for selected, passes in self.possible if selected is None] + [None] * one_maybe
233
+ least = len(sure) - one_sure + self.gone_sure
234
+ most = least + len(maybe) - one_maybe + self.gone_maybe
235
+ return RESULTS[check["assert"]](check, sure, maybe, least, most)
236
+
237
+
238
+ class Window:
239
+ """``before`` keeps every row when none matches; ``after`` keeps none."""
240
+
241
+ def __init__(self, rows: list[Row], incomplete: bool) -> None:
242
+ self.rows = rows
243
+ self.incomplete = incomplete
244
+
245
+ def kept(self, check: dict) -> list[bool | None]:
246
+ kept: list[bool | None] = [True] * len(self.rows)
247
+ for key in ORDER_KEYS:
248
+ side = self._side(key, check[key]) if key in check else kept
249
+ kept = [and3(a, b) for a, b in zip(kept, side)]
250
+ return kept
251
+
252
+ def _side(self, key: str, where: dict) -> list[bool | None]:
253
+ return {"before": self._before, "after": self._after}[key](where)
254
+
255
+ def _before(self, where: dict) -> list[bool | None]:
256
+ sure, maybe, kept = False, self.incomplete, []
257
+ for row in self.rows:
258
+ anchor = row.selected(where)
259
+ sure, maybe = sure or anchor is True, maybe or anchor is None
260
+ kept.append(BEFORE[(sure, maybe)])
261
+ return kept
262
+
263
+ def _after(self, where: dict) -> list[bool | None]:
264
+ sure, maybe, kept = False, self.incomplete, []
265
+ for row in self.rows:
266
+ kept.append(AFTER[(sure, maybe)])
267
+ anchor = row.selected(where)
268
+ sure, maybe = sure or anchor is True, maybe or anchor is None
269
+ return kept
270
+
271
+
272
+ def _rows_of(state: Mapping, entity: str) -> list[dict]:
273
+ held = state.get(entity)
274
+ if isinstance(held, dict):
275
+ return [row for row in held.values() if isinstance(row, dict)]
276
+ if isinstance(held, list):
277
+ return [row for row in held if isinstance(row, dict)]
278
+ return []
279
+
280
+
281
+ def _log_of(entity: str) -> str:
282
+ if entity == JOURNAL or entity.endswith(f".{JOURNAL}"):
283
+ return "journal"
284
+ if entity == CALL_LOG or entity.endswith(f".{CALL_LOG}"):
285
+ return "calls"
286
+ return "entity"
287
+
288
+
289
+ def _journal_meta(state: Mapping, entity: str) -> Mapping | None:
290
+ prefix = entity[: entity.rfind(".") + 1]
291
+ meta = state.get(f"{prefix}{JOURNAL_META}")
292
+ return meta if isinstance(meta, Mapping) else None
293
+
294
+
295
+ class Selection(ABC):
296
+ def __init__(self, check: dict, rows: list[Row]) -> None:
297
+ self.check = check
298
+ self.rows = rows
299
+
300
+ @staticmethod
301
+ def of(check: dict, state: Mapping) -> Selection:
302
+ entity = check["entity"]
303
+ meta = _journal_meta(state, entity)
304
+ kind = KINDS[(_log_of(entity), meta is not None)]
305
+ return kind.read(check, state, meta)
306
+
307
+ @classmethod
308
+ @abstractmethod
309
+ def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection: ...
310
+
311
+ def missing(self) -> tuple[float, float]:
312
+ return 0, 0
313
+
314
+ @abstractmethod
315
+ def undecided(self, outcome: Outcome) -> Outcome: ...
316
+
317
+ def outcome(self) -> Outcome:
318
+ incomplete = any(self.missing())
319
+ kept = Window(self.rows, incomplete).kept(self.check)
320
+ possible = [
321
+ (and3(row.selected(self.check.get("where")), inside), row.passes(self.check))
322
+ for row, inside in zip(self.rows, kept)
323
+ ]
324
+ results = Tally(possible, *self.missing()).results(self.check)
325
+ found = Outcome(
326
+ passed=False,
327
+ matched=self._seqs(possible, True),
328
+ maybe=self._seqs(possible, None),
329
+ )
330
+ if len(results) > 1:
331
+ return self.undecided(found)
332
+ return Outcome(results.pop(), None, found.matched, found.maybe)
333
+
334
+ def _seqs(self, possible: list[tuple[bool | None, bool | None]], selected: bool | None) -> tuple[int, ...]:
335
+ return tuple(
336
+ row.seq for row, (chosen, _) in zip(self.rows, possible) if chosen is selected and row.seq is not None
337
+ )
338
+
339
+
340
+ class EntitySelection(Selection):
341
+ @classmethod
342
+ def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
343
+ return cls(check, [Row(values) for values in _rows_of(state, check["entity"])])
344
+
345
+ def undecided(self, outcome: Outcome) -> Outcome:
346
+ raise GraderError(f"the rows of {self.check['entity']} cannot decide this check: ungraded")
347
+
348
+
349
+ class CallLogSelection(Selection):
350
+ def __init__(self, check: dict, rows: list[Row], dropped: int) -> None:
351
+ super().__init__(check, rows)
352
+ self.dropped = dropped
353
+
354
+ @classmethod
355
+ def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
356
+ entity = check["entity"]
357
+ rows = _rows_of(state, entity)
358
+ if _names_the_door(check) and (entity not in state or any(DOOR_FIELD not in row for row in rows)):
359
+ raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
360
+ if entity not in state:
361
+ raise GraderError(f"this engine keeps no call log ({entity})")
362
+ kept = [row["seq"] for row in rows if isinstance(row.get("seq"), int)]
363
+ return cls(check, [Row.logged(row) for row in rows], min(kept) - 1 if kept else 0)
364
+
365
+ def missing(self) -> tuple[float, float]:
366
+ narrowed = self.check.get("where") or any(key in self.check for key in ORDER_KEYS)
367
+ return (0, self.dropped) if narrowed else (self.dropped, 0)
368
+
369
+ def undecided(self, outcome: Outcome) -> Outcome:
370
+ raise GraderError(
371
+ "calls this check reads cannot say what it asks (a request too large to keep, an answer that "
372
+ "declared no rows, or a call the log no longer keeps), and the rest do not decide it: ungraded"
373
+ )
374
+
375
+
376
+ class JournalSelection(Selection):
377
+ def __init__(self, check: dict, rows: list[Row], meta: Mapping) -> None:
378
+ super().__init__(check, rows)
379
+ self.meta = meta
380
+
381
+ @classmethod
382
+ def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
383
+ assert meta is not None
384
+ return cls(check, [Row.journaled(row) for row in _rows_of(state, check["entity"])], meta)
385
+
386
+ def missing(self) -> tuple[float, float]:
387
+ return (0, 0) if self.meta.get("complete") is True else (0, math.inf)
388
+
389
+ def undecided(self, outcome: Outcome) -> Outcome:
390
+ return Outcome(False, self._reason(outcome), outcome.matched, outcome.maybe)
391
+
392
+ def _reason(self, outcome: Outcome) -> str:
393
+ if self.meta.get("complete") is not True:
394
+ why = self.meta.get("reason") or "the platform could not prove it holds every request"
395
+ return f"journal incomplete ({why}): the {len(self.rows)} requests it holds do not decide this check"
396
+ which = f"requests {', '.join(str(seq) for seq in outcome.maybe)}" if outcome.maybe else "some requests"
397
+ return (
398
+ f"{which} cannot say what this check reads (a body cut at the capture bound, or not kept), "
399
+ "and the rest do not decide it"
400
+ )
401
+
402
+
403
+ class UnrecordedSelection(Selection):
404
+ @classmethod
405
+ def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
406
+ raise GraderError(NOT_RECORDED)
407
+
408
+ def undecided(self, outcome: Outcome) -> Outcome:
409
+ raise GraderError(NOT_RECORDED)
410
+
411
+
412
+ KINDS: dict[tuple[str, bool], type[Selection]] = {
413
+ ("journal", True): JournalSelection,
414
+ ("journal", False): UnrecordedSelection,
415
+ ("calls", True): JournalSelection,
416
+ ("calls", False): CallLogSelection,
417
+ ("entity", True): EntitySelection,
418
+ ("entity", False): EntitySelection,
419
+ }
420
+
421
+
422
+ @dataclass(frozen=True)
423
+ class AssertionsGrade:
424
+ reward: float | None
425
+ rewards: dict[str, float]
426
+ reasons: dict[str, str]
427
+ outcomes: dict[str, Outcome]
428
+
429
+ def as_dict(self) -> dict:
430
+ out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
431
+ if self.reasons:
432
+ out["reasons"] = dict(self.reasons)
433
+ return out
434
+
435
+
436
+ class Assertions:
437
+ def __init__(self, checks: list[dict]) -> None:
438
+ self.checks = checks
439
+
440
+ def grade(self, state: Mapping) -> AssertionsGrade:
441
+ outcomes = {check_key(check, i): Selection.of(check, state).outcome() for i, check in enumerate(self.checks)}
442
+ weights = {check_key(check, i): check["weight"] for i, check in enumerate(self.checks)}
443
+ total = sum(weights.values())
444
+ passed = sum(weights[key] for key, outcome in outcomes.items() if outcome.passed)
445
+ return AssertionsGrade(
446
+ reward=(passed / total) if total else None,
447
+ rewards={key: 1.0 if outcome.passed else 0.0 for key, outcome in outcomes.items()},
448
+ reasons={key: outcome.reason for key, outcome in outcomes.items() if outcome.reason is not None},
449
+ outcomes=outcomes,
450
+ )
451
+
452
+
453
+ def evaluate_check(check: dict, state: Mapping) -> bool:
454
+ return Selection.of(check, state).outcome().passed
455
+
456
+
457
+ def evaluate_assertions(checks: list[dict], state: Mapping) -> dict:
458
+ return Assertions(checks).grade(state).as_dict()
@@ -23,13 +23,14 @@ import json
23
23
  import sys
24
24
  from dataclasses import dataclass, field
25
25
  from pathlib import Path
26
- from typing import IO, Any, Iterable, Iterator
26
+ from typing import IO, Any, Iterable, Iterator, NoReturn
27
27
 
28
28
  from .cast import TEXT_CASTER, EntityCasters
29
- from .compiler import Contract, quote
29
+ from .compiler import Contract, payload_value
30
30
  from .csv_rows import csv_delimiter, is_delimited_name, iter_csv_records
31
+ from .database import WorldDatabase
32
+ from .primary_key import PrimaryKey
31
33
  from .redaction import RowRedactor
32
- from .store import SQLiteStore
33
34
  from .validation import ContractError, fail, validate
34
35
 
35
36
  CHUNK_BYTES = 64 * 1024 * 1024
@@ -45,9 +46,6 @@ def encode_row(value: Any) -> str:
45
46
  return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
46
47
 
47
48
 
48
- # ---------------------------------------------------------------- streaming rows
49
-
50
-
51
49
  class _Reader:
52
50
  """A text stream with a growing buffer, for decoding JSON values one at a time.
53
51
 
@@ -256,7 +254,6 @@ def iter_file_rows(
256
254
  seen[name] = seen.get(name, 0) + 1
257
255
  yield name, row
258
256
  return
259
- # A document: rewind to its first character and stream it.
260
257
  reader.pos = start
261
258
  yield from _stream_document(reader, where, entity, seen)
262
259
 
@@ -286,10 +283,10 @@ def iter_csv_rows(
286
283
  if seen is not None:
287
284
  seen.setdefault(entity, 0)
288
285
  caster = (casters(entity) if casters else None) or TEXT_CASTER
289
- for number, record in iter_csv_records(stream, where, delimiter):
286
+ for _number, record in iter_csv_records(stream, where, delimiter):
290
287
  if seen is not None:
291
288
  seen[entity] += 1
292
- yield entity, caster.row(record, f"{where}:{number}")
289
+ yield entity, caster.row(record)
293
290
 
294
291
 
295
292
  def iter_rows(
@@ -337,9 +334,6 @@ def count_rows(
337
334
  return total
338
335
 
339
336
 
340
- # ---------------------------------------------------------------- chunks
341
-
342
-
343
337
  @dataclass
344
338
  class Chunk:
345
339
  ordinal: int
@@ -350,7 +344,7 @@ class Chunk:
350
344
  entity: str | None
351
345
  content_bytes: int # the JSONL before gzip
352
346
  content_sha256: str
353
- first_row: int # index of the chunk's first row in the input stream
347
+ first_row: int
354
348
 
355
349
  def manifest_entry(self) -> dict:
356
350
  return {
@@ -464,9 +458,6 @@ def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tup
464
458
  fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
465
459
 
466
460
 
467
- # ---------------------------------------------------------------- refusals
468
-
469
-
470
461
  def _field_of(error: ContractError) -> str | None:
471
462
  path = error.detail.get("path") or ""
472
463
  inside = path[1:] if path.startswith("/") and len(path) > 1 else None
@@ -489,14 +480,13 @@ def refusal(
489
480
  them all, ``field`` the property refused (None for a row-level refusal)."""
490
481
  key = None
491
482
  if contract is not None and isinstance(row, dict) and entity in contract.entities:
492
- fields = contract.entities[entity]["primary_key"]
493
- if all(name in row for name in fields):
494
- key = {name: row[name] for name in fields}
483
+ primary = PrimaryKey.in_contract(contract, entity)
484
+ key = primary.key(row) if primary.missing(row) is None else None
495
485
  pointer = "/" + (entity or "")
496
486
  if index is not None:
497
487
  pointer += f"/{index}"
498
488
  if field_name:
499
- pointer += f"/{field_name}"
489
+ pointer += field_name if field_name.startswith("/") else f"/{field_name}"
500
490
  return {
501
491
  "chunk": chunk,
502
492
  "index": index,
@@ -523,9 +513,9 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
523
513
  if not isinstance(row, dict):
524
514
  return refusal(chunk, index, entity, row, None, "A row must be an object")
525
515
  spec = contract.entities[entity]
526
- for name in spec["primary_key"]:
527
- if name not in row:
528
- return refusal(chunk, index, entity, row, name, f"Row is missing primary key field {name!r}", contract)
516
+ missing = PrimaryKey.of(spec).missing(row)
517
+ if missing is not None:
518
+ return refusal(chunk, index, entity, row, missing, f"Row is missing primary key field {missing!r}", contract)
529
519
  try:
530
520
  validate(spec["schema"], row, location=f"{spec['location']}/schema")
531
521
  except ContractError as error:
@@ -533,6 +523,26 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
533
523
  return None
534
524
 
535
525
 
526
+ def raise_shape_refusal(
527
+ contract: Contract, first: dict, row: Any, summary: str, context: dict, unknown_at: str
528
+ ) -> NoReturn:
529
+ """The first :func:`shape_refusal` of a file as the ``ContractError`` the store
530
+ raises for it, under its own code (``entity``, ``rows``, ``primary_key``, or the
531
+ validator's), with ``summary`` before its message and ``context`` attached."""
532
+ spec = contract.entities.get(first["entity"])
533
+ if spec is None:
534
+ fail("entity", summary + first["detail"], unknown_at, **context)
535
+ if not isinstance(row, dict):
536
+ fail("rows", summary + first["detail"], first["pointer"], **context)
537
+ if PrimaryKey.of(spec).missing(row) is not None:
538
+ fail("primary_key", summary + first["detail"], first["pointer"], **context)
539
+ try:
540
+ validate(spec["schema"], row, location=f"{spec['location']}/schema")
541
+ except ContractError as error:
542
+ raise error.contextualize(message=summary + error.detail["message"], **context)
543
+ fail("rows", summary + first["detail"], first["pointer"], **context)
544
+
545
+
536
546
  def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = REPORT_CAP) -> int:
537
547
  """Refusal records as JSONL, at most ``cap`` of them followed by
538
548
  ``{"truncated": true, "total": N}`` when there were more. Returns the total."""
@@ -546,9 +556,6 @@ def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = RE
546
556
  return total
547
557
 
548
558
 
549
- # ---------------------------------------------------------------- batches
550
-
551
-
552
559
  @dataclass
553
560
  class ChunkSpec:
554
561
  ordinal: int
@@ -560,8 +567,8 @@ class ChunkSpec:
560
567
  class _Staged:
561
568
  """The valid rows of a batch after the in-batch merge, keyed the way the store keys."""
562
569
 
563
- rows: dict[str, dict[str, dict]] = field(default_factory=dict) # entity -> key -> row
564
- origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict) # entity -> key -> (chunk, index)
570
+ rows: dict[str, dict[str, dict]] = field(default_factory=dict)
571
+ origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict)
565
572
  refusals: list[dict] = field(default_factory=list)
566
573
  read: int = 0
567
574
 
@@ -602,7 +609,7 @@ def _redacted(
602
609
 
603
610
  def _stage(
604
611
  contract: Contract,
605
- store: SQLiteStore,
612
+ store: WorldDatabase,
606
613
  chunks: Iterable[ChunkSpec],
607
614
  merge: str,
608
615
  staged: _Staged | None = None,
@@ -621,7 +628,7 @@ def _stage(
621
628
  if refused is not None:
622
629
  staged.refusals.append(refused)
623
630
  continue
624
- key = store._key(entity, row)
631
+ key = store.key_of(entity, row)
625
632
  held = staged.rows.setdefault(entity, {})
626
633
  origin = staged.origin.setdefault(entity, {})
627
634
  if key in held and merge == "fill":
@@ -635,7 +642,7 @@ def _stage(
635
642
  return staged
636
643
 
637
644
 
638
- def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged, mode: str) -> list[dict]:
645
+ def _reference_refusals(contract: Contract, store: WorldDatabase, staged: _Staged, mode: str) -> list[dict]:
639
646
  """Every row whose relationship points at a key neither the parent snapshot
640
647
  (minus replaced entities) nor the batch holds."""
641
648
  refusals: list[dict] = []
@@ -645,8 +652,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
645
652
  if target not in known:
646
653
  held = set(staged.rows.get(target, {}))
647
654
  if not (mode == "replace" and target in staged.rows):
648
- with store._trusted():
649
- held |= {row[0] for row in store._conn.execute(f"SELECT _key FROM {quote(target)}")}
655
+ held |= store.keys(target)
650
656
  known[target] = held
651
657
  return known[target]
652
658
 
@@ -657,7 +663,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
657
663
  dropped = []
658
664
  for key, row in rows.items():
659
665
  for rel in relationships:
660
- values = [row.get(name) for name in rel["fields"]]
666
+ values = [payload_value(row, name) for name in rel["fields"]]
661
667
  if all(value is None for value in values):
662
668
  continue
663
669
  chunk, index = staged.origin[entity][key]
@@ -677,7 +683,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
677
683
  break
678
684
  target = rel["target"]
679
685
  try:
680
- wanted = store._key(target, dict(zip(contract.entities[target]["primary_key"], values)))
686
+ wanted = store.key_of(target, dict(zip(contract.entities[target]["primary_key"], values)))
681
687
  except ContractError as error:
682
688
  refusals.append(
683
689
  refusal(chunk, index, entity, row, rel["fields"][0], error.detail["message"], contract)
@@ -704,7 +710,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
704
710
 
705
711
 
706
712
  def _apply(
707
- contract: Contract, store: SQLiteStore, staged: _Staged, mode: str, batch_id: str
713
+ contract: Contract, store: WorldDatabase, staged: _Staged, mode: str, batch_id: str
708
714
  ) -> tuple[int, dict | None]:
709
715
  """Apply the staged rows as one batch; the count applied, or the store's refusal."""
710
716
  from .data import seed_batch
@@ -779,7 +785,7 @@ def build_batch(
779
785
  applied_chunks: list[int] = []
780
786
  applied_digests: list[str] = []
781
787
  counts: dict[str, int] = {}
782
- with SQLiteStore.open(parent, contract, audit=False) as store:
788
+ with WorldDatabase.open(parent, contract, audit=False) as store:
783
789
  if atomic:
784
790
  staged = _stage(contract, store, chunks, merge, redactor=redactor)
785
791
  refusals.extend(staged.refusals)
@@ -800,8 +806,7 @@ def build_batch(
800
806
  found.append(refused)
801
807
  refusals.extend(found)
802
808
  if found and not dry_run:
803
- # The store rolled the chunk's transaction back; the rows it did
804
- # accept are gone with it, so the chunk is refused as a whole.
809
+ # The store rolled the chunk's transaction back, so the chunk is refused as a whole.
805
810
  continue
806
811
  if not dry_run:
807
812
  applied_rows += applied