@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
"""Pure Python over the state dump, so a check cannot read outside its session. Over a log a row may not say
|
|
2
|
+
everything a check asks, so a predicate is three-valued: a check its rows cannot decide fails, with the reason, over
|
|
3
|
+
the journal, and is ungraded over any other log."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import math
|
|
8
|
+
from abc import ABC, abstractmethod
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any, Mapping
|
|
11
|
+
|
|
12
|
+
from .journal import NOT_RECORDED
|
|
13
|
+
from .tasks import (
|
|
14
|
+
CALL_LOG,
|
|
15
|
+
DOOR_FIELD,
|
|
16
|
+
JOURNAL,
|
|
17
|
+
JOURNAL_META,
|
|
18
|
+
ORDER_KEYS,
|
|
19
|
+
ROW_FIELDS,
|
|
20
|
+
WITHHELD_FIELD,
|
|
21
|
+
GraderError,
|
|
22
|
+
_names_the_door,
|
|
23
|
+
check_key,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
RANK = {False: 0, None: 1, True: 2}
|
|
27
|
+
BEFORE = {(True, True): False, (True, False): False, (False, True): None, (False, False): True}
|
|
28
|
+
AFTER = {(True, True): True, (True, False): True, (False, True): None, (False, False): False}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def lookup(row: Any, path: str) -> tuple[bool, Any]:
|
|
32
|
+
current = row
|
|
33
|
+
for part in path.split("."):
|
|
34
|
+
if not isinstance(current, dict) or part not in current:
|
|
35
|
+
return False, None
|
|
36
|
+
current = current[part]
|
|
37
|
+
return True, current
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def same(actual: Any, expected: Any) -> bool:
|
|
41
|
+
"""JSON equality: ``true`` is not ``1``, as it is not in the schema."""
|
|
42
|
+
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
43
|
+
return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
|
|
44
|
+
return actual == expected
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def matches_where(row: dict, where: Mapping[str, Any]) -> bool:
|
|
48
|
+
return all(found and same(actual, expected) for found, actual, expected in _looked_up(row, where))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _looked_up(row: dict, where: Mapping[str, Any]) -> list[tuple[bool, Any, Any]]:
|
|
52
|
+
return [(*lookup(row, path), expected) for path, expected in where.items()]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def comparable(actual: Any, expected: Any) -> bool:
|
|
56
|
+
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
57
|
+
return False
|
|
58
|
+
numbers = (int, float)
|
|
59
|
+
if isinstance(actual, numbers) and isinstance(expected, numbers):
|
|
60
|
+
return True
|
|
61
|
+
return isinstance(actual, str) and isinstance(expected, str)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _contains(actual: Any, expected: Any) -> bool:
|
|
65
|
+
if isinstance(actual, str):
|
|
66
|
+
return isinstance(expected, str) and expected in actual
|
|
67
|
+
if isinstance(actual, list):
|
|
68
|
+
return any(same(item, expected) for item in actual)
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
ORDERED_OPS = {
|
|
73
|
+
"gt": lambda actual, expected: actual > expected,
|
|
74
|
+
"gte": lambda actual, expected: actual >= expected,
|
|
75
|
+
"lt": lambda actual, expected: actual < expected,
|
|
76
|
+
"lte": lambda actual, expected: actual <= expected,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def predicate(row: dict, field: str, op: str, expected: Any) -> bool:
|
|
81
|
+
found, actual = lookup(row, field)
|
|
82
|
+
if op == "neq":
|
|
83
|
+
return not found or not same(actual, expected)
|
|
84
|
+
if not found:
|
|
85
|
+
return False
|
|
86
|
+
if op == "eq":
|
|
87
|
+
return same(actual, expected)
|
|
88
|
+
if op == "in":
|
|
89
|
+
return any(same(actual, option) for option in expected)
|
|
90
|
+
if op == "contains":
|
|
91
|
+
return _contains(actual, expected)
|
|
92
|
+
return comparable(actual, expected) and ORDERED_OPS[op](actual, expected)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def and3(*values: bool | None) -> bool | None:
|
|
96
|
+
"""False wins over unknown (None), unknown over true."""
|
|
97
|
+
return min(values, key=RANK.__getitem__)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _under(path: str, root: str) -> bool:
|
|
101
|
+
return path == root or path.startswith(f"{root}.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _body_state(part: dict) -> str:
|
|
105
|
+
if part.get("truncated") is True:
|
|
106
|
+
return "prefix" if part.get("body") is not None else "unknown"
|
|
107
|
+
if part.get("body") is None and (part.get("bodyBytes") or 0) > 0:
|
|
108
|
+
return "unknown"
|
|
109
|
+
return "whole"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _body_paths(values: dict) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
|
113
|
+
parts = [(f"{side}.body", values.get(side)) for side in ("request", "response")]
|
|
114
|
+
states = [(path, _body_state(part)) for path, part in parts if isinstance(part, dict)]
|
|
115
|
+
unknown = tuple(path for path, state in states if state == "unknown")
|
|
116
|
+
return unknown, tuple(path for path, state in states if state == "prefix")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True)
|
|
120
|
+
class Row:
|
|
121
|
+
values: dict
|
|
122
|
+
unknown: tuple[str, ...] = ()
|
|
123
|
+
partial: tuple[str, ...] = ()
|
|
124
|
+
|
|
125
|
+
@classmethod
|
|
126
|
+
def logged(cls, values: dict) -> Row:
|
|
127
|
+
withheld = tuple(values.get(WITHHELD_FIELD) or ())
|
|
128
|
+
return cls(values, withheld + (ROW_FIELDS if values.get("found") is None else ()))
|
|
129
|
+
|
|
130
|
+
@classmethod
|
|
131
|
+
def journaled(cls, values: dict) -> Row:
|
|
132
|
+
logged = cls.logged(values)
|
|
133
|
+
door = (DOOR_FIELD,) if values.get(DOOR_FIELD) is None else ()
|
|
134
|
+
unknown, partial = _body_paths(values)
|
|
135
|
+
return cls(values, logged.unknown + door + unknown, partial)
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def seq(self) -> int | None:
|
|
139
|
+
seq = self.values.get("seq")
|
|
140
|
+
return seq if isinstance(seq, int) and not isinstance(seq, bool) else None
|
|
141
|
+
|
|
142
|
+
def cannot_say(self, path: str) -> bool:
|
|
143
|
+
if any(_under(path, root) for root in self.unknown):
|
|
144
|
+
return True
|
|
145
|
+
root = next((root for root in self.partial if _under(path, root)), None)
|
|
146
|
+
if root is None:
|
|
147
|
+
return False
|
|
148
|
+
found, value = lookup(self.values, path)
|
|
149
|
+
return path == root or not found or isinstance(value, (dict, list))
|
|
150
|
+
|
|
151
|
+
def growable(self, path: str) -> bool:
|
|
152
|
+
value = lookup(self.values, path)[1]
|
|
153
|
+
return isinstance(value, list) or (isinstance(value, str) and path in self.partial)
|
|
154
|
+
|
|
155
|
+
def selected(self, where: Mapping[str, Any] | None) -> bool | None:
|
|
156
|
+
wanted = where or {}
|
|
157
|
+
known = {path: value for path, value in wanted.items() if not self.cannot_say(path)}
|
|
158
|
+
if not matches_where(self.values, known):
|
|
159
|
+
return False
|
|
160
|
+
return None if len(known) < len(wanted) else True
|
|
161
|
+
|
|
162
|
+
def passes(self, check: dict) -> bool | None:
|
|
163
|
+
"""A withheld list or a cut body still holds what it holds, so a ``contains`` it already satisfies passes."""
|
|
164
|
+
field = check.get("field")
|
|
165
|
+
if field is None:
|
|
166
|
+
return None
|
|
167
|
+
holds = predicate(self.values, field, check["op"], check["value"])
|
|
168
|
+
if not self.cannot_say(field):
|
|
169
|
+
return holds
|
|
170
|
+
return True if holds and check["op"] == "contains" and self.growable(field) else None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@dataclass(frozen=True)
|
|
174
|
+
class Outcome:
|
|
175
|
+
passed: bool
|
|
176
|
+
reason: str | None = None
|
|
177
|
+
matched: tuple[int, ...] = ()
|
|
178
|
+
maybe: tuple[int, ...] = ()
|
|
179
|
+
|
|
180
|
+
def as_dict(self) -> dict:
|
|
181
|
+
out: dict[str, Any] = {"passed": self.passed, "matched": list(self.matched), "maybe": list(self.maybe)}
|
|
182
|
+
if self.reason is not None:
|
|
183
|
+
out["reason"] = self.reason
|
|
184
|
+
return out
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _interval(check: dict) -> tuple[float, float]:
|
|
188
|
+
assertion = check["assert"]
|
|
189
|
+
if assertion == "exists":
|
|
190
|
+
return 1, math.inf
|
|
191
|
+
if assertion == "absent":
|
|
192
|
+
return 0, 0
|
|
193
|
+
bound = check["count"]
|
|
194
|
+
return max(bound.get("gte", 0), bound.get("eq", 0)), min(bound.get("lte", math.inf), bound.get("eq", math.inf))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _counted(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
|
|
198
|
+
low, high = _interval(check)
|
|
199
|
+
can_true = max(low, least) <= min(high, most)
|
|
200
|
+
can_false = not (low <= least and most <= high)
|
|
201
|
+
return _possible_results(can_true, can_false)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _every(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
|
|
205
|
+
can_true = False not in sure and (least > 0 or any(passes is not False for passes in maybe))
|
|
206
|
+
can_false = least == 0 or any(passes is not True for passes in sure + maybe)
|
|
207
|
+
return _possible_results(can_true, can_false)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _some(check: dict, sure: list, maybe: list, least: float, most: float) -> set[bool]:
|
|
211
|
+
can_true = any(passes is not False for passes in sure + maybe)
|
|
212
|
+
can_false = True not in sure
|
|
213
|
+
return _possible_results(can_true, can_false)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _possible_results(can_true: bool, can_false: bool) -> set[bool]:
|
|
217
|
+
return {outcome for outcome, holds in ((True, can_true), (False, can_false)) if holds}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
RESULTS = {"exists": _counted, "absent": _counted, "count": _counted, "all": _every, "any": _some}
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@dataclass(frozen=True)
|
|
224
|
+
class Tally:
|
|
225
|
+
possible: list[tuple[bool | None, bool | None]]
|
|
226
|
+
gone_sure: float = 0
|
|
227
|
+
gone_maybe: float = 0
|
|
228
|
+
|
|
229
|
+
def results(self, check: dict) -> set[bool]:
|
|
230
|
+
one_sure, one_maybe = min(self.gone_sure, 1), min(self.gone_maybe, 1)
|
|
231
|
+
sure = [passes for selected, passes in self.possible if selected is True] + [None] * one_sure
|
|
232
|
+
maybe = [passes for selected, passes in self.possible if selected is None] + [None] * one_maybe
|
|
233
|
+
least = len(sure) - one_sure + self.gone_sure
|
|
234
|
+
most = least + len(maybe) - one_maybe + self.gone_maybe
|
|
235
|
+
return RESULTS[check["assert"]](check, sure, maybe, least, most)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class Window:
|
|
239
|
+
"""``before`` keeps every row when none matches; ``after`` keeps none."""
|
|
240
|
+
|
|
241
|
+
def __init__(self, rows: list[Row], incomplete: bool) -> None:
|
|
242
|
+
self.rows = rows
|
|
243
|
+
self.incomplete = incomplete
|
|
244
|
+
|
|
245
|
+
def kept(self, check: dict) -> list[bool | None]:
|
|
246
|
+
kept: list[bool | None] = [True] * len(self.rows)
|
|
247
|
+
for key in ORDER_KEYS:
|
|
248
|
+
side = self._side(key, check[key]) if key in check else kept
|
|
249
|
+
kept = [and3(a, b) for a, b in zip(kept, side)]
|
|
250
|
+
return kept
|
|
251
|
+
|
|
252
|
+
def _side(self, key: str, where: dict) -> list[bool | None]:
|
|
253
|
+
return {"before": self._before, "after": self._after}[key](where)
|
|
254
|
+
|
|
255
|
+
def _before(self, where: dict) -> list[bool | None]:
|
|
256
|
+
sure, maybe, kept = False, self.incomplete, []
|
|
257
|
+
for row in self.rows:
|
|
258
|
+
anchor = row.selected(where)
|
|
259
|
+
sure, maybe = sure or anchor is True, maybe or anchor is None
|
|
260
|
+
kept.append(BEFORE[(sure, maybe)])
|
|
261
|
+
return kept
|
|
262
|
+
|
|
263
|
+
def _after(self, where: dict) -> list[bool | None]:
|
|
264
|
+
sure, maybe, kept = False, self.incomplete, []
|
|
265
|
+
for row in self.rows:
|
|
266
|
+
kept.append(AFTER[(sure, maybe)])
|
|
267
|
+
anchor = row.selected(where)
|
|
268
|
+
sure, maybe = sure or anchor is True, maybe or anchor is None
|
|
269
|
+
return kept
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _rows_of(state: Mapping, entity: str) -> list[dict]:
|
|
273
|
+
held = state.get(entity)
|
|
274
|
+
if isinstance(held, dict):
|
|
275
|
+
return [row for row in held.values() if isinstance(row, dict)]
|
|
276
|
+
if isinstance(held, list):
|
|
277
|
+
return [row for row in held if isinstance(row, dict)]
|
|
278
|
+
return []
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _log_of(entity: str) -> str:
|
|
282
|
+
if entity == JOURNAL or entity.endswith(f".{JOURNAL}"):
|
|
283
|
+
return "journal"
|
|
284
|
+
if entity == CALL_LOG or entity.endswith(f".{CALL_LOG}"):
|
|
285
|
+
return "calls"
|
|
286
|
+
return "entity"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _journal_meta(state: Mapping, entity: str) -> Mapping | None:
|
|
290
|
+
prefix = entity[: entity.rfind(".") + 1]
|
|
291
|
+
meta = state.get(f"{prefix}{JOURNAL_META}")
|
|
292
|
+
return meta if isinstance(meta, Mapping) else None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class Selection(ABC):
|
|
296
|
+
def __init__(self, check: dict, rows: list[Row]) -> None:
|
|
297
|
+
self.check = check
|
|
298
|
+
self.rows = rows
|
|
299
|
+
|
|
300
|
+
@staticmethod
|
|
301
|
+
def of(check: dict, state: Mapping) -> Selection:
|
|
302
|
+
entity = check["entity"]
|
|
303
|
+
meta = _journal_meta(state, entity)
|
|
304
|
+
kind = KINDS[(_log_of(entity), meta is not None)]
|
|
305
|
+
return kind.read(check, state, meta)
|
|
306
|
+
|
|
307
|
+
@classmethod
|
|
308
|
+
@abstractmethod
|
|
309
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection: ...
|
|
310
|
+
|
|
311
|
+
def missing(self) -> tuple[float, float]:
|
|
312
|
+
return 0, 0
|
|
313
|
+
|
|
314
|
+
@abstractmethod
|
|
315
|
+
def undecided(self, outcome: Outcome) -> Outcome: ...
|
|
316
|
+
|
|
317
|
+
def outcome(self) -> Outcome:
|
|
318
|
+
incomplete = any(self.missing())
|
|
319
|
+
kept = Window(self.rows, incomplete).kept(self.check)
|
|
320
|
+
possible = [
|
|
321
|
+
(and3(row.selected(self.check.get("where")), inside), row.passes(self.check))
|
|
322
|
+
for row, inside in zip(self.rows, kept)
|
|
323
|
+
]
|
|
324
|
+
results = Tally(possible, *self.missing()).results(self.check)
|
|
325
|
+
found = Outcome(
|
|
326
|
+
passed=False,
|
|
327
|
+
matched=self._seqs(possible, True),
|
|
328
|
+
maybe=self._seqs(possible, None),
|
|
329
|
+
)
|
|
330
|
+
if len(results) > 1:
|
|
331
|
+
return self.undecided(found)
|
|
332
|
+
return Outcome(results.pop(), None, found.matched, found.maybe)
|
|
333
|
+
|
|
334
|
+
def _seqs(self, possible: list[tuple[bool | None, bool | None]], selected: bool | None) -> tuple[int, ...]:
|
|
335
|
+
return tuple(
|
|
336
|
+
row.seq for row, (chosen, _) in zip(self.rows, possible) if chosen is selected and row.seq is not None
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class EntitySelection(Selection):
|
|
341
|
+
@classmethod
|
|
342
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
343
|
+
return cls(check, [Row(values) for values in _rows_of(state, check["entity"])])
|
|
344
|
+
|
|
345
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
346
|
+
raise GraderError(f"the rows of {self.check['entity']} cannot decide this check: ungraded")
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class CallLogSelection(Selection):
|
|
350
|
+
def __init__(self, check: dict, rows: list[Row], dropped: int) -> None:
|
|
351
|
+
super().__init__(check, rows)
|
|
352
|
+
self.dropped = dropped
|
|
353
|
+
|
|
354
|
+
@classmethod
|
|
355
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
356
|
+
entity = check["entity"]
|
|
357
|
+
rows = _rows_of(state, entity)
|
|
358
|
+
if _names_the_door(check) and (entity not in state or any(DOOR_FIELD not in row for row in rows)):
|
|
359
|
+
raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
|
|
360
|
+
if entity not in state:
|
|
361
|
+
raise GraderError(f"this engine keeps no call log ({entity})")
|
|
362
|
+
kept = [row["seq"] for row in rows if isinstance(row.get("seq"), int)]
|
|
363
|
+
return cls(check, [Row.logged(row) for row in rows], min(kept) - 1 if kept else 0)
|
|
364
|
+
|
|
365
|
+
def missing(self) -> tuple[float, float]:
|
|
366
|
+
narrowed = self.check.get("where") or any(key in self.check for key in ORDER_KEYS)
|
|
367
|
+
return (0, self.dropped) if narrowed else (self.dropped, 0)
|
|
368
|
+
|
|
369
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
370
|
+
raise GraderError(
|
|
371
|
+
"calls this check reads cannot say what it asks (a request too large to keep, an answer that "
|
|
372
|
+
"declared no rows, or a call the log no longer keeps), and the rest do not decide it: ungraded"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
class JournalSelection(Selection):
|
|
377
|
+
def __init__(self, check: dict, rows: list[Row], meta: Mapping) -> None:
|
|
378
|
+
super().__init__(check, rows)
|
|
379
|
+
self.meta = meta
|
|
380
|
+
|
|
381
|
+
@classmethod
|
|
382
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
383
|
+
assert meta is not None
|
|
384
|
+
return cls(check, [Row.journaled(row) for row in _rows_of(state, check["entity"])], meta)
|
|
385
|
+
|
|
386
|
+
def missing(self) -> tuple[float, float]:
|
|
387
|
+
return (0, 0) if self.meta.get("complete") is True else (0, math.inf)
|
|
388
|
+
|
|
389
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
390
|
+
return Outcome(False, self._reason(outcome), outcome.matched, outcome.maybe)
|
|
391
|
+
|
|
392
|
+
def _reason(self, outcome: Outcome) -> str:
|
|
393
|
+
if self.meta.get("complete") is not True:
|
|
394
|
+
why = self.meta.get("reason") or "the platform could not prove it holds every request"
|
|
395
|
+
return f"journal incomplete ({why}): the {len(self.rows)} requests it holds do not decide this check"
|
|
396
|
+
which = f"requests {', '.join(str(seq) for seq in outcome.maybe)}" if outcome.maybe else "some requests"
|
|
397
|
+
return (
|
|
398
|
+
f"{which} cannot say what this check reads (a body cut at the capture bound, or not kept), "
|
|
399
|
+
"and the rest do not decide it"
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class UnrecordedSelection(Selection):
|
|
404
|
+
@classmethod
|
|
405
|
+
def read(cls, check: dict, state: Mapping, meta: Mapping | None) -> Selection:
|
|
406
|
+
raise GraderError(NOT_RECORDED)
|
|
407
|
+
|
|
408
|
+
def undecided(self, outcome: Outcome) -> Outcome:
|
|
409
|
+
raise GraderError(NOT_RECORDED)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
KINDS: dict[tuple[str, bool], type[Selection]] = {
|
|
413
|
+
("journal", True): JournalSelection,
|
|
414
|
+
("journal", False): UnrecordedSelection,
|
|
415
|
+
("calls", True): JournalSelection,
|
|
416
|
+
("calls", False): CallLogSelection,
|
|
417
|
+
("entity", True): EntitySelection,
|
|
418
|
+
("entity", False): EntitySelection,
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
@dataclass(frozen=True)
|
|
423
|
+
class AssertionsGrade:
|
|
424
|
+
reward: float | None
|
|
425
|
+
rewards: dict[str, float]
|
|
426
|
+
reasons: dict[str, str]
|
|
427
|
+
outcomes: dict[str, Outcome]
|
|
428
|
+
|
|
429
|
+
def as_dict(self) -> dict:
|
|
430
|
+
out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
|
|
431
|
+
if self.reasons:
|
|
432
|
+
out["reasons"] = dict(self.reasons)
|
|
433
|
+
return out
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
class Assertions:
|
|
437
|
+
def __init__(self, checks: list[dict]) -> None:
|
|
438
|
+
self.checks = checks
|
|
439
|
+
|
|
440
|
+
def grade(self, state: Mapping) -> AssertionsGrade:
|
|
441
|
+
outcomes = {check_key(check, i): Selection.of(check, state).outcome() for i, check in enumerate(self.checks)}
|
|
442
|
+
weights = {check_key(check, i): check["weight"] for i, check in enumerate(self.checks)}
|
|
443
|
+
total = sum(weights.values())
|
|
444
|
+
passed = sum(weights[key] for key, outcome in outcomes.items() if outcome.passed)
|
|
445
|
+
return AssertionsGrade(
|
|
446
|
+
reward=(passed / total) if total else None,
|
|
447
|
+
rewards={key: 1.0 if outcome.passed else 0.0 for key, outcome in outcomes.items()},
|
|
448
|
+
reasons={key: outcome.reason for key, outcome in outcomes.items() if outcome.reason is not None},
|
|
449
|
+
outcomes=outcomes,
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def evaluate_check(check: dict, state: Mapping) -> bool:
|
|
454
|
+
return Selection.of(check, state).outcome().passed
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def evaluate_assertions(checks: list[dict], state: Mapping) -> dict:
|
|
458
|
+
return Assertions(checks).grade(state).as_dict()
|
|
@@ -23,13 +23,14 @@ import json
|
|
|
23
23
|
import sys
|
|
24
24
|
from dataclasses import dataclass, field
|
|
25
25
|
from pathlib import Path
|
|
26
|
-
from typing import IO, Any, Iterable, Iterator
|
|
26
|
+
from typing import IO, Any, Iterable, Iterator, NoReturn
|
|
27
27
|
|
|
28
28
|
from .cast import TEXT_CASTER, EntityCasters
|
|
29
|
-
from .compiler import Contract,
|
|
29
|
+
from .compiler import Contract, payload_value
|
|
30
30
|
from .csv_rows import csv_delimiter, is_delimited_name, iter_csv_records
|
|
31
|
+
from .database import WorldDatabase
|
|
32
|
+
from .primary_key import PrimaryKey
|
|
31
33
|
from .redaction import RowRedactor
|
|
32
|
-
from .store import SQLiteStore
|
|
33
34
|
from .validation import ContractError, fail, validate
|
|
34
35
|
|
|
35
36
|
CHUNK_BYTES = 64 * 1024 * 1024
|
|
@@ -45,9 +46,6 @@ def encode_row(value: Any) -> str:
|
|
|
45
46
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
|
|
46
47
|
|
|
47
48
|
|
|
48
|
-
# ---------------------------------------------------------------- streaming rows
|
|
49
|
-
|
|
50
|
-
|
|
51
49
|
class _Reader:
|
|
52
50
|
"""A text stream with a growing buffer, for decoding JSON values one at a time.
|
|
53
51
|
|
|
@@ -256,7 +254,6 @@ def iter_file_rows(
|
|
|
256
254
|
seen[name] = seen.get(name, 0) + 1
|
|
257
255
|
yield name, row
|
|
258
256
|
return
|
|
259
|
-
# A document: rewind to its first character and stream it.
|
|
260
257
|
reader.pos = start
|
|
261
258
|
yield from _stream_document(reader, where, entity, seen)
|
|
262
259
|
|
|
@@ -286,10 +283,10 @@ def iter_csv_rows(
|
|
|
286
283
|
if seen is not None:
|
|
287
284
|
seen.setdefault(entity, 0)
|
|
288
285
|
caster = (casters(entity) if casters else None) or TEXT_CASTER
|
|
289
|
-
for
|
|
286
|
+
for _number, record in iter_csv_records(stream, where, delimiter):
|
|
290
287
|
if seen is not None:
|
|
291
288
|
seen[entity] += 1
|
|
292
|
-
yield entity, caster.row(record
|
|
289
|
+
yield entity, caster.row(record)
|
|
293
290
|
|
|
294
291
|
|
|
295
292
|
def iter_rows(
|
|
@@ -337,9 +334,6 @@ def count_rows(
|
|
|
337
334
|
return total
|
|
338
335
|
|
|
339
336
|
|
|
340
|
-
# ---------------------------------------------------------------- chunks
|
|
341
|
-
|
|
342
|
-
|
|
343
337
|
@dataclass
|
|
344
338
|
class Chunk:
|
|
345
339
|
ordinal: int
|
|
@@ -350,7 +344,7 @@ class Chunk:
|
|
|
350
344
|
entity: str | None
|
|
351
345
|
content_bytes: int # the JSONL before gzip
|
|
352
346
|
content_sha256: str
|
|
353
|
-
first_row: int
|
|
347
|
+
first_row: int
|
|
354
348
|
|
|
355
349
|
def manifest_entry(self) -> dict:
|
|
356
350
|
return {
|
|
@@ -464,9 +458,6 @@ def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tup
|
|
|
464
458
|
fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
|
|
465
459
|
|
|
466
460
|
|
|
467
|
-
# ---------------------------------------------------------------- refusals
|
|
468
|
-
|
|
469
|
-
|
|
470
461
|
def _field_of(error: ContractError) -> str | None:
|
|
471
462
|
path = error.detail.get("path") or ""
|
|
472
463
|
inside = path[1:] if path.startswith("/") and len(path) > 1 else None
|
|
@@ -489,14 +480,13 @@ def refusal(
|
|
|
489
480
|
them all, ``field`` the property refused (None for a row-level refusal)."""
|
|
490
481
|
key = None
|
|
491
482
|
if contract is not None and isinstance(row, dict) and entity in contract.entities:
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
key = {name: row[name] for name in fields}
|
|
483
|
+
primary = PrimaryKey.in_contract(contract, entity)
|
|
484
|
+
key = primary.key(row) if primary.missing(row) is None else None
|
|
495
485
|
pointer = "/" + (entity or "")
|
|
496
486
|
if index is not None:
|
|
497
487
|
pointer += f"/{index}"
|
|
498
488
|
if field_name:
|
|
499
|
-
pointer += f"/{field_name}"
|
|
489
|
+
pointer += field_name if field_name.startswith("/") else f"/{field_name}"
|
|
500
490
|
return {
|
|
501
491
|
"chunk": chunk,
|
|
502
492
|
"index": index,
|
|
@@ -523,9 +513,9 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
|
|
|
523
513
|
if not isinstance(row, dict):
|
|
524
514
|
return refusal(chunk, index, entity, row, None, "A row must be an object")
|
|
525
515
|
spec = contract.entities[entity]
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
516
|
+
missing = PrimaryKey.of(spec).missing(row)
|
|
517
|
+
if missing is not None:
|
|
518
|
+
return refusal(chunk, index, entity, row, missing, f"Row is missing primary key field {missing!r}", contract)
|
|
529
519
|
try:
|
|
530
520
|
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
531
521
|
except ContractError as error:
|
|
@@ -533,6 +523,26 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
|
|
|
533
523
|
return None
|
|
534
524
|
|
|
535
525
|
|
|
526
|
+
def raise_shape_refusal(
|
|
527
|
+
contract: Contract, first: dict, row: Any, summary: str, context: dict, unknown_at: str
|
|
528
|
+
) -> NoReturn:
|
|
529
|
+
"""The first :func:`shape_refusal` of a file as the ``ContractError`` the store
|
|
530
|
+
raises for it, under its own code (``entity``, ``rows``, ``primary_key``, or the
|
|
531
|
+
validator's), with ``summary`` before its message and ``context`` attached."""
|
|
532
|
+
spec = contract.entities.get(first["entity"])
|
|
533
|
+
if spec is None:
|
|
534
|
+
fail("entity", summary + first["detail"], unknown_at, **context)
|
|
535
|
+
if not isinstance(row, dict):
|
|
536
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
537
|
+
if PrimaryKey.of(spec).missing(row) is not None:
|
|
538
|
+
fail("primary_key", summary + first["detail"], first["pointer"], **context)
|
|
539
|
+
try:
|
|
540
|
+
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
541
|
+
except ContractError as error:
|
|
542
|
+
raise error.contextualize(message=summary + error.detail["message"], **context)
|
|
543
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
544
|
+
|
|
545
|
+
|
|
536
546
|
def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = REPORT_CAP) -> int:
|
|
537
547
|
"""Refusal records as JSONL, at most ``cap`` of them followed by
|
|
538
548
|
``{"truncated": true, "total": N}`` when there were more. Returns the total."""
|
|
@@ -546,9 +556,6 @@ def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = RE
|
|
|
546
556
|
return total
|
|
547
557
|
|
|
548
558
|
|
|
549
|
-
# ---------------------------------------------------------------- batches
|
|
550
|
-
|
|
551
|
-
|
|
552
559
|
@dataclass
|
|
553
560
|
class ChunkSpec:
|
|
554
561
|
ordinal: int
|
|
@@ -560,8 +567,8 @@ class ChunkSpec:
|
|
|
560
567
|
class _Staged:
|
|
561
568
|
"""The valid rows of a batch after the in-batch merge, keyed the way the store keys."""
|
|
562
569
|
|
|
563
|
-
rows: dict[str, dict[str, dict]] = field(default_factory=dict)
|
|
564
|
-
origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict)
|
|
570
|
+
rows: dict[str, dict[str, dict]] = field(default_factory=dict)
|
|
571
|
+
origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict)
|
|
565
572
|
refusals: list[dict] = field(default_factory=list)
|
|
566
573
|
read: int = 0
|
|
567
574
|
|
|
@@ -602,7 +609,7 @@ def _redacted(
|
|
|
602
609
|
|
|
603
610
|
def _stage(
|
|
604
611
|
contract: Contract,
|
|
605
|
-
store:
|
|
612
|
+
store: WorldDatabase,
|
|
606
613
|
chunks: Iterable[ChunkSpec],
|
|
607
614
|
merge: str,
|
|
608
615
|
staged: _Staged | None = None,
|
|
@@ -621,7 +628,7 @@ def _stage(
|
|
|
621
628
|
if refused is not None:
|
|
622
629
|
staged.refusals.append(refused)
|
|
623
630
|
continue
|
|
624
|
-
key = store.
|
|
631
|
+
key = store.key_of(entity, row)
|
|
625
632
|
held = staged.rows.setdefault(entity, {})
|
|
626
633
|
origin = staged.origin.setdefault(entity, {})
|
|
627
634
|
if key in held and merge == "fill":
|
|
@@ -635,7 +642,7 @@ def _stage(
|
|
|
635
642
|
return staged
|
|
636
643
|
|
|
637
644
|
|
|
638
|
-
def _reference_refusals(contract: Contract, store:
|
|
645
|
+
def _reference_refusals(contract: Contract, store: WorldDatabase, staged: _Staged, mode: str) -> list[dict]:
|
|
639
646
|
"""Every row whose relationship points at a key neither the parent snapshot
|
|
640
647
|
(minus replaced entities) nor the batch holds."""
|
|
641
648
|
refusals: list[dict] = []
|
|
@@ -645,8 +652,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
645
652
|
if target not in known:
|
|
646
653
|
held = set(staged.rows.get(target, {}))
|
|
647
654
|
if not (mode == "replace" and target in staged.rows):
|
|
648
|
-
|
|
649
|
-
held |= {row[0] for row in store._conn.execute(f"SELECT _key FROM {quote(target)}")}
|
|
655
|
+
held |= store.keys(target)
|
|
650
656
|
known[target] = held
|
|
651
657
|
return known[target]
|
|
652
658
|
|
|
@@ -657,7 +663,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
657
663
|
dropped = []
|
|
658
664
|
for key, row in rows.items():
|
|
659
665
|
for rel in relationships:
|
|
660
|
-
values = [row
|
|
666
|
+
values = [payload_value(row, name) for name in rel["fields"]]
|
|
661
667
|
if all(value is None for value in values):
|
|
662
668
|
continue
|
|
663
669
|
chunk, index = staged.origin[entity][key]
|
|
@@ -677,7 +683,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
677
683
|
break
|
|
678
684
|
target = rel["target"]
|
|
679
685
|
try:
|
|
680
|
-
wanted = store.
|
|
686
|
+
wanted = store.key_of(target, dict(zip(contract.entities[target]["primary_key"], values)))
|
|
681
687
|
except ContractError as error:
|
|
682
688
|
refusals.append(
|
|
683
689
|
refusal(chunk, index, entity, row, rel["fields"][0], error.detail["message"], contract)
|
|
@@ -704,7 +710,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
704
710
|
|
|
705
711
|
|
|
706
712
|
def _apply(
|
|
707
|
-
contract: Contract, store:
|
|
713
|
+
contract: Contract, store: WorldDatabase, staged: _Staged, mode: str, batch_id: str
|
|
708
714
|
) -> tuple[int, dict | None]:
|
|
709
715
|
"""Apply the staged rows as one batch; the count applied, or the store's refusal."""
|
|
710
716
|
from .data import seed_batch
|
|
@@ -779,7 +785,7 @@ def build_batch(
|
|
|
779
785
|
applied_chunks: list[int] = []
|
|
780
786
|
applied_digests: list[str] = []
|
|
781
787
|
counts: dict[str, int] = {}
|
|
782
|
-
with
|
|
788
|
+
with WorldDatabase.open(parent, contract, audit=False) as store:
|
|
783
789
|
if atomic:
|
|
784
790
|
staged = _stage(contract, store, chunks, merge, redactor=redactor)
|
|
785
791
|
refusals.extend(staged.refusals)
|
|
@@ -800,8 +806,7 @@ def build_batch(
|
|
|
800
806
|
found.append(refused)
|
|
801
807
|
refusals.extend(found)
|
|
802
808
|
if found and not dry_run:
|
|
803
|
-
# The store rolled the chunk's transaction back
|
|
804
|
-
# accept are gone with it, so the chunk is refused as a whole.
|
|
809
|
+
# The store rolled the chunk's transaction back, so the chunk is refused as a whole.
|
|
805
810
|
continue
|
|
806
811
|
if not dry_run:
|
|
807
812
|
applied_rows += applied
|