@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -0,0 +1,438 @@
1
+ """Bodies are what the ledger stored, redacted when captured: never re-read from anywhere else."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import uuid
7
+ from collections.abc import Callable, Iterable, Iterator, Mapping
8
+ from contextlib import contextmanager
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import TYPE_CHECKING, Any
12
+
13
+ from .._toml import tomllib
14
+ from .call_redaction import REDACTED, CallRedaction
15
+ from .database.base import SideTable
16
+ from .tasks import JOURNAL, JOURNAL_META, WITHHELD_FIELD
17
+ from .validation import fail
18
+
19
+ if TYPE_CHECKING:
20
+ from .connector import Spec
21
+ from .database import WorldDatabase
22
+
23
+ JOURNAL_KEYS = (JOURNAL, JOURNAL_META)
24
+ JOURNAL_SCHEMA = "_gateway_journal"
25
+ JOURNAL_TABLES = frozenset(JOURNAL_KEYS)
26
+ MAX_JOURNAL_REQUESTS = 20_000
27
+ NOT_RECORDED = "not recorded: only a World Host session records the requests its world receives"
28
+ LINKED_NOT_RECORDED = "not recorded for a linked world: the root world's verifiers and the task's checks read it"
29
+ VIAS = ("api", "ui", "tool", "webhook")
30
+ JSON_HINTS = ("json",)
31
+ WITHHELD_ROUTE_BODY = ["body", "selectors"]
32
+ WITHHELD_TOOL_ARGS = ["args", "selectors"]
33
+ NONE = type(None)
34
+ NUMBER = (int, float)
35
+
36
+
37
+ def _is(value: Any, kinds: tuple[type, ...]) -> bool:
38
+ """A bool is never an int here, as it is not in JSON."""
39
+ return type(value) in kinds or (float in kinds and type(value) is int)
40
+
41
+
42
+ def _check(entry: Mapping, field: str, kinds: tuple[type, ...], at: str) -> None:
43
+ if not _is(entry.get(field), kinds):
44
+ fail("journal", f"a journal request's {field} is {' or '.join(k.__name__ for k in kinds)}", f"{at}/{field}")
45
+
46
+
47
+ ENTRY_SHAPE: dict[str, tuple[type, ...]] = {
48
+ "id": (str,),
49
+ "seq": (int,),
50
+ "at": (str,),
51
+ "operation": (str, NONE),
52
+ "method": (str, NONE),
53
+ "path": (str, NONE),
54
+ "via": (str, NONE),
55
+ "rows": (int, NONE),
56
+ "found": (bool, NONE),
57
+ "durationMs": (float, NONE),
58
+ "query": (dict,),
59
+ "request": (dict,),
60
+ "response": (dict,),
61
+ }
62
+ BODY_SHAPE: dict[str, tuple[type, ...]] = {
63
+ "body": (str, NONE),
64
+ "bodyBytes": (int, NONE),
65
+ "truncated": (bool,),
66
+ "contentType": (str, NONE),
67
+ }
68
+
69
+
70
+ def _strings_in(node: Any) -> list[str]:
71
+ found: dict[str, None] = {}
72
+ stack = [node]
73
+ while stack:
74
+ item = stack.pop()
75
+ if isinstance(item, str) and item != REDACTED:
76
+ found[item] = None
77
+ if isinstance(item, dict):
78
+ stack.extend(reversed(list(item.values())))
79
+ if isinstance(item, list):
80
+ stack.extend(reversed(item))
81
+ return list(found)
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class JournalBody:
86
+ value: Any
87
+ whole: bool
88
+
89
+ @classmethod
90
+ def of(cls, part: Mapping) -> JournalBody:
91
+ text = part.get("body")
92
+ whole = part.get("truncated") is not True and (text is not None or not part.get("bodyBytes"))
93
+ return cls(None if text is None else cls._read(text, part.get("contentType")), whole)
94
+
95
+ @staticmethod
96
+ def _read(text: str, content_type: Any) -> Any:
97
+ typed = isinstance(content_type, str) and any(hint in content_type.lower() for hint in JSON_HINTS)
98
+ if not typed and not text.lstrip().startswith(("{", "[")):
99
+ return text
100
+ try:
101
+ return json.loads(text)
102
+ except (ValueError, RecursionError):
103
+ return text
104
+
105
+
106
+ @dataclass(frozen=True)
107
+ class JournalEntry:
108
+ """Checked: the host trusts the platform no more than the platform trusts it."""
109
+
110
+ raw: dict
111
+
112
+ @classmethod
113
+ def parse(cls, value: Any, at: str) -> JournalEntry:
114
+ if not isinstance(value, dict):
115
+ fail("journal", "a journal request is an object", at)
116
+ for field, kinds in ENTRY_SHAPE.items():
117
+ _check(value, field, kinds, at)
118
+ for side in ("request", "response"):
119
+ cls._part(value[side], f"{at}/{side}")
120
+ _check(value["response"], "status", (int,), f"{at}/response")
121
+ if value["via"] is not None and value["via"] not in VIAS:
122
+ fail("journal", f"a journal request came in by one of {', '.join(VIAS)}", f"{at}/via")
123
+ withheld = value.get(WITHHELD_FIELD, [])
124
+ if not isinstance(withheld, list) or any(not isinstance(path, str) for path in withheld):
125
+ fail("journal", "withheld lists the paths of a request the platform did not send", f"{at}/withheld")
126
+ return cls(value)
127
+
128
+ @staticmethod
129
+ def _part(part: dict, at: str) -> None:
130
+ for field, kinds in BODY_SHAPE.items():
131
+ _check(part, field, kinds, at)
132
+ headers = part.get("headers", {})
133
+ if not isinstance(headers, dict) or any(not isinstance(v, str) for v in headers.values()):
134
+ fail("journal", "headers are {name: value}", f"{at}/headers")
135
+
136
+ @property
137
+ def seq(self) -> int:
138
+ return self.raw["seq"]
139
+
140
+ @property
141
+ def operation(self) -> str | None:
142
+ return self.raw["operation"]
143
+
144
+ @property
145
+ def request(self) -> JournalBody:
146
+ return JournalBody.of(self.raw["request"])
147
+
148
+ def row(self) -> dict:
149
+ raw, request, response = self.raw, self.raw["request"], self.raw["response"]
150
+ out = {
151
+ key: raw[key]
152
+ for key in ("seq", "id", "at", "durationMs", "via", "operation", "method", "path", "query", "rows", "found")
153
+ }
154
+ out["request"] = {"headers": dict(request.get("headers") or {}), **self._shown(request)}
155
+ out["response"] = {"status": response["status"], **self._shown(response)}
156
+ if raw.get(WITHHELD_FIELD):
157
+ out[WITHHELD_FIELD] = list(raw[WITHHELD_FIELD])
158
+ return out
159
+
160
+ @staticmethod
161
+ def _shown(part: dict) -> dict:
162
+ return {
163
+ "body": JournalBody.of(part).value,
164
+ "bodyBytes": part.get("bodyBytes"),
165
+ "truncated": part.get("truncated") is True,
166
+ "contentType": part.get("contentType"),
167
+ }
168
+
169
+ def cut(self, columns: frozenset[str]) -> bool:
170
+ withheld = set(self.raw.get(WITHHELD_FIELD) or ())
171
+ parts = {BODY_COLUMNS[column] for column in columns if column in BODY_COLUMNS}
172
+ return any(part in withheld or self._truncated(part) for part in parts)
173
+
174
+ def _truncated(self, part: str) -> bool:
175
+ side = part.partition(".")[0]
176
+ return part.endswith(".body") and self.raw[side].get("truncated") is True
177
+
178
+ def answered(self) -> dict:
179
+ raw = self.raw
180
+ return {
181
+ "via": raw["via"],
182
+ "status": raw["response"]["status"],
183
+ "rows": raw["rows"],
184
+ "found": raw["found"],
185
+ "elapsed_ms": raw["durationMs"],
186
+ }
187
+
188
+
189
+ @dataclass(frozen=True)
190
+ class WorldJournal:
191
+ entries: tuple[JournalEntry, ...]
192
+ complete: bool
193
+ reason: str | None = None
194
+ recorded: bool = True
195
+
196
+ @classmethod
197
+ def unrecorded(cls, reason: str = NOT_RECORDED) -> WorldJournal:
198
+ return cls((), False, reason, recorded=False)
199
+
200
+ @classmethod
201
+ def of(cls, value: Any) -> WorldJournal | None:
202
+ return None if value is None else cls.parse(value)
203
+
204
+ @classmethod
205
+ def parse(cls, value: Any, at: str = "/journal") -> WorldJournal:
206
+ if not isinstance(value, dict) or set(value) - {"complete", "reason", "requests"}:
207
+ fail("journal", "a journal is {complete, reason?, requests: [request, ...]}", at)
208
+ requests = value.get("requests")
209
+ if not isinstance(requests, list) or len(requests) > MAX_JOURNAL_REQUESTS:
210
+ fail("journal", f"a journal's requests are a list of at most {MAX_JOURNAL_REQUESTS}", f"{at}/requests")
211
+ if not isinstance(value.get("complete"), bool):
212
+ fail("journal", "complete is true or false", f"{at}/complete")
213
+ if not _is(value.get("reason"), (str, NONE)):
214
+ fail("journal", "reason is text", f"{at}/reason")
215
+ entries = tuple(JournalEntry.parse(entry, f"{at}/requests/{i}") for i, entry in enumerate(requests))
216
+ if any(later.seq <= earlier.seq for earlier, later in zip(entries, entries[1:])):
217
+ fail("journal", "a journal's requests come in increasing seq", f"{at}/requests")
218
+ return cls(entries, value["complete"], value.get("reason"))
219
+
220
+ def rows(self) -> list[dict]:
221
+ return [entry.row() for entry in self.entries]
222
+
223
+ def meta(self) -> dict:
224
+ return {
225
+ "complete": self.complete,
226
+ "reason": self.reason,
227
+ "requests": len(self.entries),
228
+ "withheld": sum(1 for entry in self.entries if entry.raw.get(WITHHELD_FIELD)),
229
+ }
230
+
231
+ def gap(self, use: JournalUse) -> str | None:
232
+ if not use.tables:
233
+ return None
234
+ if not self.complete:
235
+ return f"journal incomplete ({self.reason or 'it may be missing requests'})"
236
+ cut = sum(1 for entry in self.entries if entry.cut(use.columns))
237
+ return f"the bodies of {cut} requests it reads were withheld or cut short" if cut else None
238
+
239
+ def into_state(self, state: dict, calls: list[dict]) -> dict:
240
+ return {**state, JOURNAL: self.rows(), JOURNAL_META: self.meta(), "calls": calls}
241
+
242
+
243
+ LINKED_JOURNAL = WorldJournal.unrecorded(LINKED_NOT_RECORDED)
244
+
245
+
246
+ class CallsView:
247
+ """A request no route took, a tool call the door refused and a webhook the world sent (``via``
248
+ webhook) are the journal's alone: they never were the agent's calls."""
249
+
250
+ def __init__(self, spec: Spec | None, redaction: CallRedaction, admitted: Callable[[str], bool]) -> None:
251
+ self.spec = spec
252
+ self.redaction = redaction
253
+ self.admitted = admitted
254
+
255
+ def rows(self, journal: WorldJournal) -> list[dict]:
256
+ return [call for call in map(self._call, journal.entries) if call is not None]
257
+
258
+ def _call(self, entry: JournalEntry) -> dict | None:
259
+ if entry.operation is None or entry.raw["via"] == "webhook":
260
+ return None
261
+ if entry.raw["via"] == "tool" and entry.raw["method"] is None:
262
+ return self._tool(entry) if self.admitted(entry.operation) else None
263
+ return self._route(entry)
264
+
265
+ def _tool(self, entry: JournalEntry) -> dict:
266
+ body = entry.request
267
+ args = body.value if body.whole else REDACTED
268
+ withheld = {} if body.whole else {WITHHELD_FIELD: list(WITHHELD_TOOL_ARGS)}
269
+ return {
270
+ **self._heading(entry),
271
+ "method": None,
272
+ "path": None,
273
+ "args": args,
274
+ "query": {},
275
+ "body": None,
276
+ "selectors": _strings_in(args) if body.whole else [],
277
+ **entry.answered(),
278
+ **withheld,
279
+ }
280
+
281
+ def _route(self, entry: JournalEntry) -> dict:
282
+ body = entry.request
283
+ kept = body.value if body.whole and isinstance(body.value, (dict, list)) and body.value else None
284
+ args = self._args(entry)
285
+ query = entry.raw["query"]
286
+ withheld = {} if body.whole else {WITHHELD_FIELD: list(WITHHELD_ROUTE_BODY)}
287
+ return {
288
+ **self._heading(entry),
289
+ "method": entry.raw["method"],
290
+ "path": entry.raw["path"],
291
+ "args": args,
292
+ "query": query,
293
+ "body": kept,
294
+ "selectors": _strings_in([args, query, kept]) if body.whole else _strings_in([args, query]),
295
+ **entry.answered(),
296
+ **withheld,
297
+ }
298
+
299
+ @staticmethod
300
+ def _heading(entry: JournalEntry) -> dict:
301
+ return {"seq": entry.seq, "at": entry.raw["at"], "operation": entry.operation}
302
+
303
+ def _args(self, entry: JournalEntry) -> dict:
304
+ operations = self.spec.operations if self.spec is not None else {}
305
+ operation = operations.get(entry.operation or "")
306
+ path = entry.raw["path"]
307
+ if operation is None or not isinstance(path, str):
308
+ return {}
309
+ return self.redaction.value(operation.path_args(path))
310
+
311
+
312
+ JOURNAL_COLUMNS = (
313
+ ("seq", "INTEGER"),
314
+ ("id", "TEXT"),
315
+ ("at", "TEXT"),
316
+ ("duration_ms", "REAL"),
317
+ ("via", "TEXT"),
318
+ ("operation", "TEXT"),
319
+ ("method", "TEXT"),
320
+ ("path", "TEXT"),
321
+ ("query", "TEXT"),
322
+ ("status", "INTEGER"),
323
+ ("rows", "INTEGER"),
324
+ ("found", "INTEGER"),
325
+ ("request_headers", "TEXT"),
326
+ ("request_body", "TEXT"),
327
+ ("request_body_bytes", "INTEGER"),
328
+ ("request_truncated", "INTEGER"),
329
+ ("request_content_type", "TEXT"),
330
+ ("response_body", "TEXT"),
331
+ ("response_body_bytes", "INTEGER"),
332
+ ("response_truncated", "INTEGER"),
333
+ ("response_content_type", "TEXT"),
334
+ ("withheld", "TEXT"),
335
+ )
336
+ JOURNAL_META_COLUMNS = (("complete", "INTEGER"), ("reason", "TEXT"), ("requests", "INTEGER"), ("withheld", "INTEGER"))
337
+ BODY_COLUMNS = {
338
+ "request_headers": "request.headers",
339
+ "request_body": "request.body",
340
+ "response_body": "response.body",
341
+ }
342
+
343
+
344
+ @dataclass(frozen=True)
345
+ class JournalUse:
346
+ tables: bool
347
+ columns: frozenset[str] = frozenset()
348
+
349
+ @classmethod
350
+ def none(cls) -> JournalUse:
351
+ return cls(False)
352
+
353
+
354
+ def _flag(value: Any) -> int | None:
355
+ return None if value is None else int(bool(value))
356
+
357
+
358
+ def _table_row(entry: JournalEntry) -> tuple:
359
+ raw, request, response = entry.raw, entry.raw["request"], entry.raw["response"]
360
+ return (
361
+ raw["seq"],
362
+ raw["id"],
363
+ raw["at"],
364
+ raw["durationMs"],
365
+ raw["via"],
366
+ raw["operation"],
367
+ raw["method"],
368
+ raw["path"],
369
+ json.dumps(raw["query"], ensure_ascii=False),
370
+ response["status"],
371
+ raw["rows"],
372
+ _flag(raw["found"]),
373
+ json.dumps(request.get("headers") or {}, ensure_ascii=False),
374
+ request.get("body"),
375
+ request.get("bodyBytes"),
376
+ _flag(request.get("truncated")),
377
+ request.get("contentType"),
378
+ response.get("body"),
379
+ response.get("bodyBytes"),
380
+ _flag(response.get("truncated")),
381
+ response.get("contentType"),
382
+ json.dumps(raw[WITHHELD_FIELD]) if raw.get(WITHHELD_FIELD) else None,
383
+ )
384
+
385
+
386
+ class JournalDatabase:
387
+ """A read the store's authorizer does not report (a NATURAL or USING join's count) still counts when a statement
388
+ names a journal table or body column, so the verifier still fails closed."""
389
+
390
+ def __init__(self, journal: WorldJournal, scratch: Path) -> None:
391
+ self.journal = journal
392
+ self.scratch = Path(scratch)
393
+
394
+ def tables(self) -> tuple[SideTable, ...]:
395
+ meta = self.journal.meta()
396
+ summary = (int(meta["complete"]), meta["reason"], meta["requests"], meta["withheld"])
397
+ return (
398
+ SideTable(JOURNAL, JOURNAL_COLUMNS, tuple(map(_table_row, self.journal.entries))),
399
+ SideTable(JOURNAL_META, JOURNAL_META_COLUMNS, (summary,)),
400
+ )
401
+
402
+ @contextmanager
403
+ def attached(self, store: WorldDatabase) -> Iterator[Callable[[list[str]], JournalUse]]:
404
+ self.scratch.mkdir(parents=True, exist_ok=True)
405
+ path = self.scratch / f"journal-{uuid.uuid4().hex}"
406
+ try:
407
+ store.write_side_tables(path, self.tables())
408
+ store.attach(JOURNAL_SCHEMA, path, JOURNAL_TABLES, bare=True)
409
+ except BaseException:
410
+ path.unlink(missing_ok=True)
411
+ raise
412
+ try:
413
+ yield lambda statements: self._use(store, statements)
414
+ finally:
415
+ try:
416
+ store.detach(JOURNAL_SCHEMA)
417
+ finally:
418
+ path.unlink(missing_ok=True)
419
+
420
+ @staticmethod
421
+ def _use(store: WorldDatabase, statements: list[str]) -> JournalUse:
422
+ named = frozenset().union(*(store.identifiers(statement) for statement in statements))
423
+ tables = store.reads_of(JOURNAL_SCHEMA) > 0 or bool(named & JOURNAL_TABLES)
424
+ return JournalUse(tables, store.columns_read_of(JOURNAL_SCHEMA) | (named & frozenset(BODY_COLUMNS)))
425
+
426
+
427
+ def accepting_incomplete(bundle: Path) -> frozenset[str]:
428
+ try:
429
+ manifest = tomllib.loads((Path(bundle) / "gateway-env.toml").read_text(encoding="utf-8"))
430
+ except (OSError, tomllib.TOMLDecodeError):
431
+ return frozenset()
432
+ table = manifest.get("verifiers")
433
+ entries: Iterable[tuple[str, Any]] = table.items() if isinstance(table, dict) else ()
434
+ return frozenset(
435
+ name
436
+ for name, policy in entries
437
+ if isinstance(policy, dict) and policy.get("accepts_incomplete_journal") is True
438
+ )