@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""Bodies are what the ledger stored, redacted when captured: never re-read from anywhere else."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import uuid
|
|
7
|
+
from collections.abc import Callable, Iterable, Iterator, Mapping
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING, Any
|
|
12
|
+
|
|
13
|
+
from .._toml import tomllib
|
|
14
|
+
from .call_redaction import REDACTED, CallRedaction
|
|
15
|
+
from .database.base import SideTable
|
|
16
|
+
from .tasks import JOURNAL, JOURNAL_META, WITHHELD_FIELD
|
|
17
|
+
from .validation import fail
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from .connector import Spec
|
|
21
|
+
from .database import WorldDatabase
|
|
22
|
+
|
|
23
|
+
JOURNAL_KEYS = (JOURNAL, JOURNAL_META)
|
|
24
|
+
JOURNAL_SCHEMA = "_gateway_journal"
|
|
25
|
+
JOURNAL_TABLES = frozenset(JOURNAL_KEYS)
|
|
26
|
+
MAX_JOURNAL_REQUESTS = 20_000
|
|
27
|
+
NOT_RECORDED = "not recorded: only a World Host session records the requests its world receives"
|
|
28
|
+
LINKED_NOT_RECORDED = "not recorded for a linked world: the root world's verifiers and the task's checks read it"
|
|
29
|
+
VIAS = ("api", "ui", "tool")
|
|
30
|
+
JSON_HINTS = ("json",)
|
|
31
|
+
WITHHELD_ROUTE_BODY = ["body", "selectors"]
|
|
32
|
+
WITHHELD_TOOL_ARGS = ["args", "selectors"]
|
|
33
|
+
NONE = type(None)
|
|
34
|
+
NUMBER = (int, float)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _is(value: Any, kinds: tuple[type, ...]) -> bool:
|
|
38
|
+
"""A bool is never an int here, as it is not in JSON."""
|
|
39
|
+
return type(value) in kinds or (float in kinds and type(value) is int)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _check(entry: Mapping, field: str, kinds: tuple[type, ...], at: str) -> None:
|
|
43
|
+
if not _is(entry.get(field), kinds):
|
|
44
|
+
fail("journal", f"a journal request's {field} is {' or '.join(k.__name__ for k in kinds)}", f"{at}/{field}")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
ENTRY_SHAPE: dict[str, tuple[type, ...]] = {
|
|
48
|
+
"id": (str,),
|
|
49
|
+
"seq": (int,),
|
|
50
|
+
"at": (str,),
|
|
51
|
+
"operation": (str, NONE),
|
|
52
|
+
"method": (str, NONE),
|
|
53
|
+
"path": (str, NONE),
|
|
54
|
+
"via": (str, NONE),
|
|
55
|
+
"rows": (int, NONE),
|
|
56
|
+
"found": (bool, NONE),
|
|
57
|
+
"durationMs": (float, NONE),
|
|
58
|
+
"query": (dict,),
|
|
59
|
+
"request": (dict,),
|
|
60
|
+
"response": (dict,),
|
|
61
|
+
}
|
|
62
|
+
BODY_SHAPE: dict[str, tuple[type, ...]] = {
|
|
63
|
+
"body": (str, NONE),
|
|
64
|
+
"bodyBytes": (int, NONE),
|
|
65
|
+
"truncated": (bool,),
|
|
66
|
+
"contentType": (str, NONE),
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _strings_in(node: Any) -> list[str]:
|
|
71
|
+
found: dict[str, None] = {}
|
|
72
|
+
stack = [node]
|
|
73
|
+
while stack:
|
|
74
|
+
item = stack.pop()
|
|
75
|
+
if isinstance(item, str) and item != REDACTED:
|
|
76
|
+
found[item] = None
|
|
77
|
+
if isinstance(item, dict):
|
|
78
|
+
stack.extend(reversed(list(item.values())))
|
|
79
|
+
if isinstance(item, list):
|
|
80
|
+
stack.extend(reversed(item))
|
|
81
|
+
return list(found)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class JournalBody:
|
|
86
|
+
value: Any
|
|
87
|
+
whole: bool
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def of(cls, part: Mapping) -> JournalBody:
|
|
91
|
+
text = part.get("body")
|
|
92
|
+
whole = part.get("truncated") is not True and (text is not None or not part.get("bodyBytes"))
|
|
93
|
+
return cls(None if text is None else cls._read(text, part.get("contentType")), whole)
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _read(text: str, content_type: Any) -> Any:
|
|
97
|
+
typed = isinstance(content_type, str) and any(hint in content_type.lower() for hint in JSON_HINTS)
|
|
98
|
+
if not typed and not text.lstrip().startswith(("{", "[")):
|
|
99
|
+
return text
|
|
100
|
+
try:
|
|
101
|
+
return json.loads(text)
|
|
102
|
+
except (ValueError, RecursionError):
|
|
103
|
+
return text
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass(frozen=True)
|
|
107
|
+
class JournalEntry:
|
|
108
|
+
"""Checked: the host trusts the platform no more than the platform trusts it."""
|
|
109
|
+
|
|
110
|
+
raw: dict
|
|
111
|
+
|
|
112
|
+
@classmethod
|
|
113
|
+
def parse(cls, value: Any, at: str) -> JournalEntry:
|
|
114
|
+
if not isinstance(value, dict):
|
|
115
|
+
fail("journal", "a journal request is an object", at)
|
|
116
|
+
for field, kinds in ENTRY_SHAPE.items():
|
|
117
|
+
_check(value, field, kinds, at)
|
|
118
|
+
for side in ("request", "response"):
|
|
119
|
+
cls._part(value[side], f"{at}/{side}")
|
|
120
|
+
_check(value["response"], "status", (int,), f"{at}/response")
|
|
121
|
+
if value["via"] is not None and value["via"] not in VIAS:
|
|
122
|
+
fail("journal", f"a journal request came in by one of {', '.join(VIAS)}", f"{at}/via")
|
|
123
|
+
withheld = value.get(WITHHELD_FIELD, [])
|
|
124
|
+
if not isinstance(withheld, list) or any(not isinstance(path, str) for path in withheld):
|
|
125
|
+
fail("journal", "withheld lists the paths of a request the platform did not send", f"{at}/withheld")
|
|
126
|
+
return cls(value)
|
|
127
|
+
|
|
128
|
+
@staticmethod
|
|
129
|
+
def _part(part: dict, at: str) -> None:
|
|
130
|
+
for field, kinds in BODY_SHAPE.items():
|
|
131
|
+
_check(part, field, kinds, at)
|
|
132
|
+
headers = part.get("headers", {})
|
|
133
|
+
if not isinstance(headers, dict) or any(not isinstance(v, str) for v in headers.values()):
|
|
134
|
+
fail("journal", "headers are {name: value}", f"{at}/headers")
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def seq(self) -> int:
|
|
138
|
+
return self.raw["seq"]
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def operation(self) -> str | None:
|
|
142
|
+
return self.raw["operation"]
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def request(self) -> JournalBody:
|
|
146
|
+
return JournalBody.of(self.raw["request"])
|
|
147
|
+
|
|
148
|
+
def row(self) -> dict:
|
|
149
|
+
raw, request, response = self.raw, self.raw["request"], self.raw["response"]
|
|
150
|
+
out = {
|
|
151
|
+
key: raw[key]
|
|
152
|
+
for key in ("seq", "id", "at", "durationMs", "via", "operation", "method", "path", "query", "rows", "found")
|
|
153
|
+
}
|
|
154
|
+
out["request"] = {"headers": dict(request.get("headers") or {}), **self._shown(request)}
|
|
155
|
+
out["response"] = {"status": response["status"], **self._shown(response)}
|
|
156
|
+
if raw.get(WITHHELD_FIELD):
|
|
157
|
+
out[WITHHELD_FIELD] = list(raw[WITHHELD_FIELD])
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
@staticmethod
|
|
161
|
+
def _shown(part: dict) -> dict:
|
|
162
|
+
return {
|
|
163
|
+
"body": JournalBody.of(part).value,
|
|
164
|
+
"bodyBytes": part.get("bodyBytes"),
|
|
165
|
+
"truncated": part.get("truncated") is True,
|
|
166
|
+
"contentType": part.get("contentType"),
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
def cut(self, columns: frozenset[str]) -> bool:
|
|
170
|
+
withheld = set(self.raw.get(WITHHELD_FIELD) or ())
|
|
171
|
+
parts = {BODY_COLUMNS[column] for column in columns if column in BODY_COLUMNS}
|
|
172
|
+
return any(part in withheld or self._truncated(part) for part in parts)
|
|
173
|
+
|
|
174
|
+
def _truncated(self, part: str) -> bool:
|
|
175
|
+
side = part.partition(".")[0]
|
|
176
|
+
return part.endswith(".body") and self.raw[side].get("truncated") is True
|
|
177
|
+
|
|
178
|
+
def answered(self) -> dict:
|
|
179
|
+
raw = self.raw
|
|
180
|
+
return {
|
|
181
|
+
"via": raw["via"],
|
|
182
|
+
"status": raw["response"]["status"],
|
|
183
|
+
"rows": raw["rows"],
|
|
184
|
+
"found": raw["found"],
|
|
185
|
+
"elapsed_ms": raw["durationMs"],
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@dataclass(frozen=True)
|
|
190
|
+
class WorldJournal:
|
|
191
|
+
entries: tuple[JournalEntry, ...]
|
|
192
|
+
complete: bool
|
|
193
|
+
reason: str | None = None
|
|
194
|
+
recorded: bool = True
|
|
195
|
+
|
|
196
|
+
@classmethod
|
|
197
|
+
def unrecorded(cls, reason: str = NOT_RECORDED) -> WorldJournal:
|
|
198
|
+
return cls((), False, reason, recorded=False)
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def of(cls, value: Any) -> WorldJournal | None:
|
|
202
|
+
return None if value is None else cls.parse(value)
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def parse(cls, value: Any, at: str = "/journal") -> WorldJournal:
|
|
206
|
+
if not isinstance(value, dict) or set(value) - {"complete", "reason", "requests"}:
|
|
207
|
+
fail("journal", "a journal is {complete, reason?, requests: [request, ...]}", at)
|
|
208
|
+
requests = value.get("requests")
|
|
209
|
+
if not isinstance(requests, list) or len(requests) > MAX_JOURNAL_REQUESTS:
|
|
210
|
+
fail("journal", f"a journal's requests are a list of at most {MAX_JOURNAL_REQUESTS}", f"{at}/requests")
|
|
211
|
+
if not isinstance(value.get("complete"), bool):
|
|
212
|
+
fail("journal", "complete is true or false", f"{at}/complete")
|
|
213
|
+
if not _is(value.get("reason"), (str, NONE)):
|
|
214
|
+
fail("journal", "reason is text", f"{at}/reason")
|
|
215
|
+
entries = tuple(JournalEntry.parse(entry, f"{at}/requests/{i}") for i, entry in enumerate(requests))
|
|
216
|
+
if any(later.seq <= earlier.seq for earlier, later in zip(entries, entries[1:])):
|
|
217
|
+
fail("journal", "a journal's requests come in increasing seq", f"{at}/requests")
|
|
218
|
+
return cls(entries, value["complete"], value.get("reason"))
|
|
219
|
+
|
|
220
|
+
def rows(self) -> list[dict]:
|
|
221
|
+
return [entry.row() for entry in self.entries]
|
|
222
|
+
|
|
223
|
+
def meta(self) -> dict:
|
|
224
|
+
return {
|
|
225
|
+
"complete": self.complete,
|
|
226
|
+
"reason": self.reason,
|
|
227
|
+
"requests": len(self.entries),
|
|
228
|
+
"withheld": sum(1 for entry in self.entries if entry.raw.get(WITHHELD_FIELD)),
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
def gap(self, use: JournalUse) -> str | None:
|
|
232
|
+
if not use.tables:
|
|
233
|
+
return None
|
|
234
|
+
if not self.complete:
|
|
235
|
+
return f"journal incomplete ({self.reason or 'it may be missing requests'})"
|
|
236
|
+
cut = sum(1 for entry in self.entries if entry.cut(use.columns))
|
|
237
|
+
return f"the bodies of {cut} requests it reads were withheld or cut short" if cut else None
|
|
238
|
+
|
|
239
|
+
def into_state(self, state: dict, calls: list[dict]) -> dict:
|
|
240
|
+
return {**state, JOURNAL: self.rows(), JOURNAL_META: self.meta(), "calls": calls}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
LINKED_JOURNAL = WorldJournal.unrecorded(LINKED_NOT_RECORDED)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class CallsView:
|
|
247
|
+
"""A request no route took and a tool call the door refused are the journal's alone: they never were calls."""
|
|
248
|
+
|
|
249
|
+
def __init__(self, spec: Spec | None, redaction: CallRedaction, admitted: Callable[[str], bool]) -> None:
|
|
250
|
+
self.spec = spec
|
|
251
|
+
self.redaction = redaction
|
|
252
|
+
self.admitted = admitted
|
|
253
|
+
|
|
254
|
+
def rows(self, journal: WorldJournal) -> list[dict]:
|
|
255
|
+
return [call for call in map(self._call, journal.entries) if call is not None]
|
|
256
|
+
|
|
257
|
+
def _call(self, entry: JournalEntry) -> dict | None:
|
|
258
|
+
if entry.operation is None:
|
|
259
|
+
return None
|
|
260
|
+
if entry.raw["via"] == "tool" and entry.raw["method"] is None:
|
|
261
|
+
return self._tool(entry) if self.admitted(entry.operation) else None
|
|
262
|
+
return self._route(entry)
|
|
263
|
+
|
|
264
|
+
def _tool(self, entry: JournalEntry) -> dict:
|
|
265
|
+
body = entry.request
|
|
266
|
+
args = body.value if body.whole else REDACTED
|
|
267
|
+
withheld = {} if body.whole else {WITHHELD_FIELD: list(WITHHELD_TOOL_ARGS)}
|
|
268
|
+
return {
|
|
269
|
+
**self._heading(entry),
|
|
270
|
+
"method": None,
|
|
271
|
+
"path": None,
|
|
272
|
+
"args": args,
|
|
273
|
+
"query": {},
|
|
274
|
+
"body": None,
|
|
275
|
+
"selectors": _strings_in(args) if body.whole else [],
|
|
276
|
+
**entry.answered(),
|
|
277
|
+
**withheld,
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
def _route(self, entry: JournalEntry) -> dict:
|
|
281
|
+
body = entry.request
|
|
282
|
+
kept = body.value if body.whole and isinstance(body.value, (dict, list)) and body.value else None
|
|
283
|
+
args = self._args(entry)
|
|
284
|
+
query = entry.raw["query"]
|
|
285
|
+
withheld = {} if body.whole else {WITHHELD_FIELD: list(WITHHELD_ROUTE_BODY)}
|
|
286
|
+
return {
|
|
287
|
+
**self._heading(entry),
|
|
288
|
+
"method": entry.raw["method"],
|
|
289
|
+
"path": entry.raw["path"],
|
|
290
|
+
"args": args,
|
|
291
|
+
"query": query,
|
|
292
|
+
"body": kept,
|
|
293
|
+
"selectors": _strings_in([args, query, kept]) if body.whole else _strings_in([args, query]),
|
|
294
|
+
**entry.answered(),
|
|
295
|
+
**withheld,
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
@staticmethod
|
|
299
|
+
def _heading(entry: JournalEntry) -> dict:
|
|
300
|
+
return {"seq": entry.seq, "at": entry.raw["at"], "operation": entry.operation}
|
|
301
|
+
|
|
302
|
+
def _args(self, entry: JournalEntry) -> dict:
|
|
303
|
+
operations = self.spec.operations if self.spec is not None else {}
|
|
304
|
+
operation = operations.get(entry.operation or "")
|
|
305
|
+
path = entry.raw["path"]
|
|
306
|
+
if operation is None or not isinstance(path, str):
|
|
307
|
+
return {}
|
|
308
|
+
return self.redaction.value(operation.path_args(path))
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
JOURNAL_COLUMNS = (
|
|
312
|
+
("seq", "INTEGER"),
|
|
313
|
+
("id", "TEXT"),
|
|
314
|
+
("at", "TEXT"),
|
|
315
|
+
("duration_ms", "REAL"),
|
|
316
|
+
("via", "TEXT"),
|
|
317
|
+
("operation", "TEXT"),
|
|
318
|
+
("method", "TEXT"),
|
|
319
|
+
("path", "TEXT"),
|
|
320
|
+
("query", "TEXT"),
|
|
321
|
+
("status", "INTEGER"),
|
|
322
|
+
("rows", "INTEGER"),
|
|
323
|
+
("found", "INTEGER"),
|
|
324
|
+
("request_headers", "TEXT"),
|
|
325
|
+
("request_body", "TEXT"),
|
|
326
|
+
("request_body_bytes", "INTEGER"),
|
|
327
|
+
("request_truncated", "INTEGER"),
|
|
328
|
+
("request_content_type", "TEXT"),
|
|
329
|
+
("response_body", "TEXT"),
|
|
330
|
+
("response_body_bytes", "INTEGER"),
|
|
331
|
+
("response_truncated", "INTEGER"),
|
|
332
|
+
("response_content_type", "TEXT"),
|
|
333
|
+
("withheld", "TEXT"),
|
|
334
|
+
)
|
|
335
|
+
JOURNAL_META_COLUMNS = (("complete", "INTEGER"), ("reason", "TEXT"), ("requests", "INTEGER"), ("withheld", "INTEGER"))
|
|
336
|
+
BODY_COLUMNS = {
|
|
337
|
+
"request_headers": "request.headers",
|
|
338
|
+
"request_body": "request.body",
|
|
339
|
+
"response_body": "response.body",
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@dataclass(frozen=True)
|
|
344
|
+
class JournalUse:
|
|
345
|
+
tables: bool
|
|
346
|
+
columns: frozenset[str] = frozenset()
|
|
347
|
+
|
|
348
|
+
@classmethod
|
|
349
|
+
def none(cls) -> JournalUse:
|
|
350
|
+
return cls(False)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _flag(value: Any) -> int | None:
|
|
354
|
+
return None if value is None else int(bool(value))
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _table_row(entry: JournalEntry) -> tuple:
|
|
358
|
+
raw, request, response = entry.raw, entry.raw["request"], entry.raw["response"]
|
|
359
|
+
return (
|
|
360
|
+
raw["seq"],
|
|
361
|
+
raw["id"],
|
|
362
|
+
raw["at"],
|
|
363
|
+
raw["durationMs"],
|
|
364
|
+
raw["via"],
|
|
365
|
+
raw["operation"],
|
|
366
|
+
raw["method"],
|
|
367
|
+
raw["path"],
|
|
368
|
+
json.dumps(raw["query"], ensure_ascii=False),
|
|
369
|
+
response["status"],
|
|
370
|
+
raw["rows"],
|
|
371
|
+
_flag(raw["found"]),
|
|
372
|
+
json.dumps(request.get("headers") or {}, ensure_ascii=False),
|
|
373
|
+
request.get("body"),
|
|
374
|
+
request.get("bodyBytes"),
|
|
375
|
+
_flag(request.get("truncated")),
|
|
376
|
+
request.get("contentType"),
|
|
377
|
+
response.get("body"),
|
|
378
|
+
response.get("bodyBytes"),
|
|
379
|
+
_flag(response.get("truncated")),
|
|
380
|
+
response.get("contentType"),
|
|
381
|
+
json.dumps(raw[WITHHELD_FIELD]) if raw.get(WITHHELD_FIELD) else None,
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
class JournalDatabase:
|
|
386
|
+
"""A read the store's authorizer does not report (a NATURAL or USING join's count) still counts when a statement
|
|
387
|
+
names a journal table or body column, so the verifier still fails closed."""
|
|
388
|
+
|
|
389
|
+
def __init__(self, journal: WorldJournal, scratch: Path) -> None:
|
|
390
|
+
self.journal = journal
|
|
391
|
+
self.scratch = Path(scratch)
|
|
392
|
+
|
|
393
|
+
def tables(self) -> tuple[SideTable, ...]:
|
|
394
|
+
meta = self.journal.meta()
|
|
395
|
+
summary = (int(meta["complete"]), meta["reason"], meta["requests"], meta["withheld"])
|
|
396
|
+
return (
|
|
397
|
+
SideTable(JOURNAL, JOURNAL_COLUMNS, tuple(map(_table_row, self.journal.entries))),
|
|
398
|
+
SideTable(JOURNAL_META, JOURNAL_META_COLUMNS, (summary,)),
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
@contextmanager
|
|
402
|
+
def attached(self, store: WorldDatabase) -> Iterator[Callable[[list[str]], JournalUse]]:
|
|
403
|
+
self.scratch.mkdir(parents=True, exist_ok=True)
|
|
404
|
+
path = self.scratch / f"journal-{uuid.uuid4().hex}"
|
|
405
|
+
try:
|
|
406
|
+
store.write_side_tables(path, self.tables())
|
|
407
|
+
store.attach(JOURNAL_SCHEMA, path, JOURNAL_TABLES, bare=True)
|
|
408
|
+
except BaseException:
|
|
409
|
+
path.unlink(missing_ok=True)
|
|
410
|
+
raise
|
|
411
|
+
try:
|
|
412
|
+
yield lambda statements: self._use(store, statements)
|
|
413
|
+
finally:
|
|
414
|
+
try:
|
|
415
|
+
store.detach(JOURNAL_SCHEMA)
|
|
416
|
+
finally:
|
|
417
|
+
path.unlink(missing_ok=True)
|
|
418
|
+
|
|
419
|
+
@staticmethod
|
|
420
|
+
def _use(store: WorldDatabase, statements: list[str]) -> JournalUse:
|
|
421
|
+
named = frozenset().union(*(store.identifiers(statement) for statement in statements))
|
|
422
|
+
tables = store.reads_of(JOURNAL_SCHEMA) > 0 or bool(named & JOURNAL_TABLES)
|
|
423
|
+
return JournalUse(tables, store.columns_read_of(JOURNAL_SCHEMA) | (named & frozenset(BODY_COLUMNS)))
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def accepting_incomplete(bundle: Path) -> frozenset[str]:
|
|
427
|
+
try:
|
|
428
|
+
manifest = tomllib.loads((Path(bundle) / "gateway-env.toml").read_text(encoding="utf-8"))
|
|
429
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
430
|
+
return frozenset()
|
|
431
|
+
table = manifest.get("verifiers")
|
|
432
|
+
entries: Iterable[tuple[str, Any]] = table.items() if isinstance(table, dict) else ()
|
|
433
|
+
return frozenset(
|
|
434
|
+
name
|
|
435
|
+
for name, policy in entries
|
|
436
|
+
if isinstance(policy, dict) and policy.get("accepts_incomplete_journal") is True
|
|
437
|
+
)
|