@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -3,17 +3,34 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import collections
6
+ import json
6
7
  import os
8
+ import stat
7
9
  import threading
10
+ import urllib.parse
8
11
  from dataclasses import dataclass
9
12
  from datetime import datetime, timezone
10
13
  from enum import StrEnum
11
- from typing import Any
14
+ from functools import partial
15
+ from pathlib import Path
16
+ from typing import TYPE_CHECKING, Any, Callable, Iterable
12
17
 
18
+ from .call_capture import Exchange, Served
13
19
  from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
14
20
  from .connector import Operation, Spec, _dig
21
+ from .handler_errors import log_line
15
22
  from .tasks import WITHHELD_FIELD
16
23
 
24
+ from .webhooks import ANSWER_SUFFIX, SUBSCRIPTIONS_SHOWN, WebhookAnswer, WebhookFeed, WebhookPoll
25
+
26
+ if TYPE_CHECKING:
27
+ from .webhooks import Delivery, WebhookOutbox
28
+ from .request_ledger import RequestOutbox
29
+
30
+ METHOD_CHARS = 32
31
+ OPERATION_CHARS = 512
32
+ PATH_CHARS = 64 << 10
33
+ LINE_BOUNDS = {"method": METHOD_CHARS, "operation": OPERATION_CHARS, "path": PATH_CHARS}
17
34
  DEFAULT_CALL_LOG_MAX = 5000
18
35
  DEFAULT_CALL_LOG_VALUES = 200_000
19
36
  DEFAULT_CALL_LOG_CHARS = 8 << 20
@@ -27,6 +44,35 @@ WITHHELD_ROUTE = {
27
44
  WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
28
45
 
29
46
 
47
+ def read_tail(path: str | Path, max_bytes: int | None, dir_fd: int | None = None) -> bytes:
48
+ """The newest ``max_bytes`` of the file at ``path`` (all of it when None; relative to ``dir_fd`` when
49
+ given) as tenant code could have left it: no link followed, no FIFO waited on, anything but a regular
50
+ file read as empty. Raises OSError when it cannot be opened or read; its descriptor is closed on
51
+ every path."""
52
+ descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dir_fd)
53
+ try:
54
+ found = os.fstat(descriptor)
55
+ if not stat.S_ISREG(found.st_mode):
56
+ return b""
57
+ size = found.st_size if max_bytes is None else min(max_bytes, found.st_size)
58
+ os.lseek(descriptor, found.st_size - size, os.SEEK_SET)
59
+ return _read_up_to(descriptor, size)
60
+ finally:
61
+ os.close(descriptor)
62
+
63
+
64
+ def _read_up_to(descriptor: int, max_bytes: int) -> bytes:
65
+ chunks: list[bytes] = []
66
+ left = max_bytes
67
+ while left > 0:
68
+ chunk = os.read(descriptor, min(left, 1 << 20))
69
+ if not chunk:
70
+ break
71
+ chunks.append(chunk)
72
+ left -= len(chunk)
73
+ return b"".join(chunks)
74
+
75
+
30
76
  def _children(node: Any) -> Any:
31
77
  if isinstance(node, dict):
32
78
  return node.values()
@@ -63,6 +109,21 @@ def _found(status: int, rows: int | None) -> bool | None:
63
109
  return None if rows is None else rows > 0
64
110
 
65
111
 
112
+ def _bounded(line: dict) -> dict:
113
+ """A ledger line's free-text fields cut to the ledger's bounds (``limits`` in
114
+ conformance/world-request-log.json), so the platform keeps the request."""
115
+ return {
116
+ field: value[:size] if isinstance(value, str) else value
117
+ for field, size in LINE_BOUNDS.items()
118
+ for value in [line.get(field)]
119
+ }
120
+
121
+
122
+ def _answered(status: int, rows: int | None, elapsed_s: float) -> dict:
123
+ rows = None if status >= 400 else rows
124
+ return {"status": status, "rows": rows, "found": _found(status, rows), "elapsed_ms": round(elapsed_s * 1000, 1)}
125
+
126
+
66
127
  class Via(StrEnum):
67
128
  """The door a call came in by, set by the server from where it arrived and
68
129
  never from a request header. It names the door, not the actor: a token
@@ -71,6 +132,7 @@ class Via(StrEnum):
71
132
  API = "api"
72
133
  UI = "ui"
73
134
  TOOL = "tool"
135
+ WEBHOOK = "webhook"
74
136
 
75
137
 
76
138
  @dataclass(frozen=True)
@@ -106,6 +168,93 @@ class AnswerShape:
106
168
  TOOL_ANSWER = AnswerShape()
107
169
 
108
170
 
171
+ class CallJournal:
172
+ """A version worker's session's call log on disk beside its store, one JSON line
173
+ per entry as it is kept, so a worker that dies leaves its calls with its rows and
174
+ the next worker resumes both. A call is journaled after its transaction commits:
175
+ a worker that dies in between keeps the call's rows and loses its entry. Once what
176
+ was appended since the last rewrite passes the log's own budget, the file is
177
+ rewritten from the newest of the log's entries that fit ``budget`` bytes: bounded
178
+ on disk whatever the calls carry, and the whole log unless its text is mostly
179
+ escaped in JSON (a control character takes six bytes)."""
180
+
181
+ def __init__(self, path: Path) -> None:
182
+ self.path = Path(path)
183
+ self.appended = 0
184
+
185
+ @staticmethod
186
+ def line(entry: dict) -> bytes:
187
+ return (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
188
+
189
+ def append(self, line: bytes) -> None:
190
+ try:
191
+ self.path.parent.mkdir(parents=True, exist_ok=True)
192
+ with open(self.path, "ab") as journal:
193
+ journal.write(line)
194
+ except OSError as error:
195
+ return self._unwritten(error)
196
+ self.appended += len(line)
197
+
198
+ def rewrite(self, entries: Iterable[dict], budget: int) -> None:
199
+ staging = self.path.with_name(f".{self.path.name}.next")
200
+ try:
201
+ self.path.parent.mkdir(parents=True, exist_ok=True)
202
+ staging.write_bytes(b"".join(self._newest(entries, budget)))
203
+ os.replace(staging, self.path)
204
+ except OSError as error:
205
+ return self._unwritten(error)
206
+ self.appended = 0
207
+
208
+ @classmethod
209
+ def _newest(cls, entries: Iterable[dict], budget: int) -> list[bytes]:
210
+ kept: list[bytes] = []
211
+ for line in map(cls.line, reversed(list(entries))):
212
+ budget -= len(line)
213
+ if budget < 0:
214
+ break
215
+ kept.append(line)
216
+ return kept[::-1]
217
+
218
+ def _unwritten(self, error: OSError) -> None:
219
+ log_line(f"call journal {self.path.name} not written: {error}")
220
+
221
+ @staticmethod
222
+ def read(path: Path) -> list[dict]:
223
+ """The entries a journal holds, oldest first, read by the worker that continues the session
224
+ from it: :meth:`read_left` over the whole file, so a worker killed mid-write or a tenant's
225
+ directory, link or FIFO at the path costs that journal's entries, never the reopen."""
226
+ return CallJournal._read(path, None)
227
+
228
+ @staticmethod
229
+ def read_left(path: Path, max_bytes: int) -> list[dict]:
230
+ """The newest ``max_bytes`` of a journal a dead worker left, read by the supervisor: the
231
+ worker ran tenant code, so a link, a FIFO, a directory or anything but a regular file
232
+ reads as empty, and so does a file that cannot be read."""
233
+ return CallJournal._read(path, max_bytes)
234
+
235
+ @staticmethod
236
+ def _read(path: Path, max_bytes: int | None) -> list[dict]:
237
+ try:
238
+ text = read_tail(path, max_bytes).decode("utf-8", "replace")
239
+ except OSError:
240
+ return []
241
+ return CallJournal._entries(text)
242
+
243
+ @staticmethod
244
+ def _entries(text: str) -> list[dict]:
245
+ """The entries of a journal's text, one to a line. Lines end at ``\\n`` alone: ``str.splitlines``
246
+ also cuts at U+2028, U+2029 and U+0085, which :meth:`line` leaves raw inside an entry."""
247
+ return [entry for entry in map(CallJournal._entry, text.split("\n")) if entry is not None]
248
+
249
+ @staticmethod
250
+ def _entry(line: str) -> dict | None:
251
+ try:
252
+ entry = json.loads(line)
253
+ except (ValueError, RecursionError):
254
+ return None
255
+ return entry if isinstance(entry, dict) and isinstance(entry.get("seq"), int) else None
256
+
257
+
109
258
  class CallLog:
110
259
  """The calls one session answered, in order: what the agent asked (a
111
260
  route's operation, method, path, path args, query and body; a tool's name
@@ -128,7 +277,11 @@ class CallLog:
128
277
  token, a checkpoint carries it (:meth:`resume`), and a task's
129
278
  ``verify.py`` is handed it
130
279
  as ``calls`` beside the rows, because a read-only investigation leaves no
131
- row behind and its calls are the only evidence of what it did."""
280
+ row behind and its calls are the only evidence of what it did.
281
+
282
+ ``outbox`` is the session's half of the world request ledger: every call recorded
283
+ here, and every request no route took (:meth:`heard`), goes there with its
284
+ exchange (:class:`Exchange`) whatever this log's own bounds keep."""
132
285
 
133
286
  def __init__(
134
287
  self,
@@ -136,6 +289,8 @@ class CallLog:
136
289
  redaction: CallRedaction | None = None,
137
290
  max_values: int = DEFAULT_CALL_LOG_VALUES,
138
291
  max_chars: int = DEFAULT_CALL_LOG_CHARS,
292
+ journal: CallJournal | None = None,
293
+ outbox: RequestOutbox | None = None,
139
294
  ):
140
295
  if maximum is None:
141
296
  raw = os.environ.get("WORLDS_CALL_LOG_MAX")
@@ -149,11 +304,22 @@ class CallLog:
149
304
  self.values = 0
150
305
  self.chars = 0
151
306
  self.total = 0
307
+ self.journal = journal
308
+ self.outbox = outbox
309
+ self.webhooks = WebhookFeed()
152
310
  self._lock = threading.Lock()
153
311
 
154
312
  @classmethod
155
- def for_spec(cls, spec: Spec) -> CallLog:
156
- return cls(redaction=CallRedaction.of(spec))
313
+ def for_spec(
314
+ cls,
315
+ spec: Spec | None,
316
+ journal: CallJournal | None = None,
317
+ outbox: RequestOutbox | None = None,
318
+ secret: str | None = None,
319
+ ) -> CallLog:
320
+ """The call log of one session of the world ``spec`` declares (None: a world of
321
+ tools alone), whose strings never keep ``secret``, the session's credential."""
322
+ return cls(redaction=CallRedaction.of(spec, secret), journal=journal, outbox=outbox)
157
323
 
158
324
  def record(
159
325
  self,
@@ -169,16 +335,47 @@ class CallLog:
169
335
  via: Via | None,
170
336
  *,
171
337
  shape: AnswerShape,
338
+ served: Served | None = None,
172
339
  ) -> None:
173
- if self.maximum == 0:
174
- return
175
340
  door = {} if via is None else {"via": str(via)}
176
341
  heading = {"operation": operation.name, "method": method}
177
342
  try:
178
343
  request, budget = self._route_request(operation, path, args, query_all, body)
179
344
  except (RecursionError, RequestTooLarge):
180
345
  request, budget = self._route_line(operation, path, args, query_all)
181
- self._append({**heading, **request, **door}, status, shape.rows(answer), elapsed_s, budget)
346
+ answered = _answered(status, shape.rows(answer), elapsed_s)
347
+ secret = served.secret if served else None
348
+ line = {
349
+ **heading,
350
+ "path": CallRedaction.path_without_secret(request["path"], secret),
351
+ "query": CallRedaction.without_secret(request["query"], secret),
352
+ "via": door.get("via"),
353
+ }
354
+ exchange = partial(Exchange.of, served, body, status, answer, self.redaction) if served else None
355
+ self._send(line, answered, exchange)
356
+ if self.maximum > 0:
357
+ self._append({**heading, **request, **door}, answered, budget)
358
+
359
+ def heard(
360
+ self, method: str, path: str, query: str, status: int, answer: Any, via: Via | None, served: Served
361
+ ) -> None:
362
+ """A request no route took (refused, or answered before routing): the ledger keeps it,
363
+ the grader's log does not."""
364
+ try:
365
+ kept_query = self.redaction.query({k: list(v) for k, v in urllib.parse.parse_qs(query).items()})
366
+ except (RecursionError, RequestTooLarge):
367
+ kept_query = {}
368
+ kept_query = CallRedaction.without_secret(kept_query, served.secret)
369
+ line = {"operation": None, "method": method, "path": self.redaction.raw_path(path, served.secret)}
370
+ exchange = partial(Exchange.of, served, None, status, answer, self.redaction)
371
+ door = None if via is None else str(via)
372
+ self._send({**line, "query": kept_query, "via": door}, _answered(status, None, 0.0), exchange)
373
+
374
+ def _send(self, line: dict, answered: dict, exchange: Callable[[], Exchange] | None) -> None:
375
+ if self.outbox is None or exchange is None:
376
+ return
377
+ found = {"rows": answered["rows"], "found": answered["found"], "durationMs": answered["elapsed_ms"]}
378
+ self.outbox.append({**line, **_bounded(line), **found}, exchange())
182
379
 
183
380
  def _route_request(
184
381
  self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
@@ -210,34 +407,93 @@ class CallLog:
210
407
  "query": redaction.query(query_all, budget),
211
408
  }
212
409
 
410
+ def refused_tool(self, tool: str, args: Any, refusal: dict) -> None:
411
+ """A tool call the door refused before it ran (an unknown or disallowed name): the
412
+ ledger keeps it with the refusal it was answered, the grader's log does not."""
413
+ answer = {"error": refusal.get("code"), "message": refusal.get("message")}
414
+ exchange = partial(Exchange.of_tool, args, 400, answer, self.redaction)
415
+ line = {"operation": tool, "method": None, "path": None, "query": {}, "via": str(Via.TOOL)}
416
+ self._send(line, _answered(400, None, 0.0), exchange)
417
+
213
418
  def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
419
+ heading = {"operation": tool, "method": None, "path": None}
420
+ answered = _answered(status, TOOL_ANSWER.rows(answer), elapsed_s)
421
+ exchange = partial(Exchange.of_tool, args, status, answer, self.redaction)
422
+ self._send({**heading, "query": {}, "via": str(Via.TOOL)}, answered, exchange)
214
423
  if self.maximum == 0:
215
424
  return
216
- heading = {"operation": tool, "method": None, "path": None}
217
425
  try:
218
426
  request, budget = self._tool_request(args)
219
427
  except (RecursionError, RequestTooLarge):
220
428
  request, budget = dict(WITHHELD_TOOL), RequestBudget()
221
- self._append({**heading, **request, "via": str(Via.TOOL)}, status, TOOL_ANSWER.rows(answer), elapsed_s, budget)
429
+ self._append({**heading, **request, "via": str(Via.TOOL)}, answered, budget)
430
+
431
+ def record_webhooks(self, deliveries: Iterable[Delivery]) -> None:
432
+ """Each delivery the world sent, kept by the request ledger with ``via`` webhook and no
433
+ answer yet (status 0); the grader's call log keeps only what the agent called."""
434
+ sent = list(deliveries)
435
+ self.webhooks.append(sent)
436
+ for delivery in sent:
437
+ self._send_webhook(delivery, WebhookAnswer(delivery.id, 0, None, None), {"delivery": delivery.id}, "")
438
+
439
+ def answer_webhooks(self, answers: Iterable[WebhookAnswer]) -> int:
440
+ """Each receiver's answer to a forwarded delivery, kept by the request ledger as one more
441
+ ``via`` webhook entry under ``<event type>.answer`` (so a check counting the event never counts
442
+ it) with the receiver's status and ``query.answer`` naming the delivery; the
443
+ first answer to each delivery only, and none for an id the feed no longer holds."""
444
+ kept = 0
445
+ for answer in answers:
446
+ delivery = self.webhooks.answer(answer.delivery_id)
447
+ if delivery is None:
448
+ continue
449
+ self._send_webhook(delivery, answer, {"answer": delivery.id}, ANSWER_SUFFIX)
450
+ kept += 1
451
+ return kept
452
+
453
+ def webhook_page(self, outbox: WebhookOutbox | None, poll: WebhookPoll) -> dict:
454
+ """A forwarder's poll: its answers kept first, then the subscriptions ``outbox`` reads (none
455
+ declared: ``declared`` false; at most :data:`SUBSCRIPTIONS_SHOWN`, each with how many deliveries
456
+ this feed sent it) and the page of deliveries."""
457
+ declared = outbox is not None and outbox.webhooks is not None
458
+ kept = self.answer_webhooks(poll.answers)
459
+ found = outbox.subscriptions() if outbox is not None and poll.subscriptions else []
460
+ shown = [{**sub, "sent": self.webhooks.sent_to(sub["key"])} for sub in found[:SUBSCRIPTIONS_SHOWN]]
461
+ page = self.webhooks.after(poll.after, poll.limit, poll.epoch, poll.tail, poll.targets)
462
+ listed = {"subscriptions": shown, "truncated": len(found) > len(shown)}
463
+ return {"declared": declared, **listed, "answered": kept, **page}
464
+
465
+ def _send_webhook(self, delivery: Delivery, answer: WebhookAnswer, query: dict, suffix: str) -> None:
466
+ path = self.redaction.raw_path(delivery.endpoint, self.redaction.secret)
467
+ line = {"operation": f"{delivery.event.type}{suffix}", "method": "POST", "path": path, "query": query}
468
+ exchange = partial(Exchange.of_webhook, delivery, answer, self.redaction)
469
+ answered = _answered(answer.status, None, (answer.duration_ms or 0.0) / 1000)
470
+ self._send({**line, "via": str(Via.WEBHOOK)}, answered, exchange)
222
471
 
223
472
  def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
224
473
  budget = RequestBudget()
225
474
  logged_args = self.redaction.value(args, budget)
226
475
  return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
227
476
 
228
- def _append(self, request: dict, status: int, rows: int | None, elapsed_s: float, budget: RequestBudget) -> None:
229
- rows = None if status >= 400 else rows
230
- answered = {
231
- "status": status,
232
- "rows": rows,
233
- "found": _found(status, rows),
234
- "elapsed_ms": round(elapsed_s * 1000, 1),
235
- }
477
+ def _append(self, request: dict, answered: dict, budget: RequestBudget) -> None:
236
478
  at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
237
479
  cost = budget.spent()
238
480
  with self._lock:
239
481
  self.total += 1
240
- self._keep({"seq": self.total, "at": at, **request, **answered}, cost)
482
+ entry = {"seq": self.total, "at": at, **request, **answered}
483
+ self._keep(entry, cost)
484
+ self._journal(entry)
485
+
486
+ def _journal(self, entry: dict) -> None:
487
+ if self.journal is None:
488
+ return
489
+ line = CallJournal.line(entry)
490
+ if self.journal.appended + len(line) > 2 * self.max_chars:
491
+ return self._rewrite_journal()
492
+ self.journal.append(line)
493
+
494
+ def _rewrite_journal(self) -> None:
495
+ if self.journal is not None:
496
+ self.journal.rewrite(self.entries, 4 * self.max_chars)
241
497
 
242
498
  def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
243
499
  self.entries.append(entry)
@@ -281,6 +537,7 @@ class CallLog:
281
537
  for call, cost in zip(calls, costs):
282
538
  self._keep(call, cost)
283
539
  self.total = max([self.total, *(call["seq"] for call in calls)])
540
+ self._rewrite_journal()
284
541
 
285
542
  def clear(self) -> None:
286
543
  with self._lock:
@@ -289,3 +546,4 @@ class CallLog:
289
546
  self.values = 0
290
547
  self.chars = 0
291
548
  self.total = 0
549
+ self._rewrite_journal()