@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,503 @@
1
+ """The call log: every call one session answered, by every door, in order."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import collections
6
+ import json
7
+ import os
8
+ import stat
9
+ import threading
10
+ import urllib.parse
11
+ from dataclasses import dataclass
12
+ from datetime import datetime, timezone
13
+ from enum import StrEnum
14
+ from functools import partial
15
+ from pathlib import Path
16
+ from typing import TYPE_CHECKING, Any, Callable, Iterable
17
+
18
+ from .call_capture import Exchange, Served
19
+ from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
20
+ from .connector import Operation, Spec, _dig
21
+ from .handler_errors import log_line
22
+ from .tasks import WITHHELD_FIELD
23
+
24
+ if TYPE_CHECKING:
25
+ from .request_ledger import RequestOutbox
26
+
27
+ METHOD_CHARS = 32
28
+ OPERATION_CHARS = 512
29
+ PATH_CHARS = 64 << 10
30
+ LINE_BOUNDS = {"method": METHOD_CHARS, "operation": OPERATION_CHARS, "path": PATH_CHARS}
31
+ DEFAULT_CALL_LOG_MAX = 5000
32
+ DEFAULT_CALL_LOG_VALUES = 200_000
33
+ DEFAULT_CALL_LOG_CHARS = 8 << 20
34
+ WITHHELD_ROUTE = {
35
+ "args": REDACTED,
36
+ "query": {},
37
+ "body": None,
38
+ "selectors": [],
39
+ WITHHELD_FIELD: ["path", "args", "query", "body", "selectors"],
40
+ }
41
+ WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
42
+
43
+
44
+ def read_tail(path: str | Path, max_bytes: int | None, dir_fd: int | None = None) -> bytes:
45
+ """The newest ``max_bytes`` of the file at ``path`` (all of it when None; relative to ``dir_fd`` when
46
+ given) as tenant code could have left it: no link followed, no FIFO waited on, anything but a regular
47
+ file read as empty. Raises OSError when it cannot be opened or read; its descriptor is closed on
48
+ every path."""
49
+ descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dir_fd)
50
+ try:
51
+ found = os.fstat(descriptor)
52
+ if not stat.S_ISREG(found.st_mode):
53
+ return b""
54
+ size = found.st_size if max_bytes is None else min(max_bytes, found.st_size)
55
+ os.lseek(descriptor, found.st_size - size, os.SEEK_SET)
56
+ return _read_up_to(descriptor, size)
57
+ finally:
58
+ os.close(descriptor)
59
+
60
+
61
+ def _read_up_to(descriptor: int, max_bytes: int) -> bytes:
62
+ chunks: list[bytes] = []
63
+ left = max_bytes
64
+ while left > 0:
65
+ chunk = os.read(descriptor, min(left, 1 << 20))
66
+ if not chunk:
67
+ break
68
+ chunks.append(chunk)
69
+ left -= len(chunk)
70
+ return b"".join(chunks)
71
+
72
+
73
+ def _children(node: Any) -> Any:
74
+ if isinstance(node, dict):
75
+ return node.values()
76
+ if isinstance(node, (list, tuple)):
77
+ return node
78
+ return ()
79
+
80
+
81
+ def _strings_in(node: Any) -> list[str]:
82
+ """Every string in a JSON value, however nested, once each, in document order."""
83
+ found: dict[str, None] = {}
84
+ _collect_strings(node, found)
85
+ return list(found)
86
+
87
+
88
+ def _collect_strings(node: Any, found: dict[str, None]) -> None:
89
+ if isinstance(node, str) and node != REDACTED:
90
+ found[node] = None
91
+ for value in _children(node):
92
+ _collect_strings(value, found)
93
+
94
+
95
+ def _counted(found: Any) -> int | None:
96
+ if isinstance(found, list):
97
+ return len(found)
98
+ if isinstance(found, dict):
99
+ return 1 if found else 0
100
+ return 0 if found is None else None
101
+
102
+
103
+ def _found(status: int, rows: int | None) -> bool | None:
104
+ if status >= 400:
105
+ return False
106
+ return None if rows is None else rows > 0
107
+
108
+
109
+ def _bounded(line: dict) -> dict:
110
+ """A ledger line's free-text fields cut to the ledger's bounds (``limits`` in
111
+ conformance/world-request-log.json), so the platform keeps the request."""
112
+ return {
113
+ field: value[:size] if isinstance(value, str) else value
114
+ for field, size in LINE_BOUNDS.items()
115
+ for value in [line.get(field)]
116
+ }
117
+
118
+
119
+ def _answered(status: int, rows: int | None, elapsed_s: float) -> dict:
120
+ rows = None if status >= 400 else rows
121
+ return {"status": status, "rows": rows, "found": _found(status, rows), "elapsed_ms": round(elapsed_s * 1000, 1)}
122
+
123
+
124
+ class Via(StrEnum):
125
+ """The door a call came in by, set by the server from where it arrived and
126
+ never from a request header. It names the door, not the actor: a token
127
+ holder calling the ui origin's prefix without a browser is ``ui``."""
128
+
129
+ API = "api"
130
+ UI = "ui"
131
+ TOOL = "tool"
132
+
133
+
134
+ @dataclass(frozen=True)
135
+ class AnswerShape:
136
+ """Where an answer carries its rows, as the route declares it: a list is
137
+ its rows; under the envelope key the route serves, a list; at the
138
+ operation's results path, a list, one object (one row) or nothing (none);
139
+ a ``single`` route's object is one row. Nothing answered is no rows. Any
140
+ other answer (a tool's object, a handler's own shape) carries rows nobody
141
+ declared: unknown, ``None``."""
142
+
143
+ envelope: str = ""
144
+ results: str = ""
145
+ single: bool = False
146
+
147
+ def rows(self, answer: Any) -> int | None:
148
+ if isinstance(answer, list):
149
+ return len(answer)
150
+ if answer is None:
151
+ return 0
152
+ if not isinstance(answer, dict):
153
+ return None
154
+ listed = _dig(answer, self.envelope) if self.envelope else None
155
+ if isinstance(listed, list):
156
+ return len(listed)
157
+ if self.results:
158
+ return _counted(_dig(answer, self.results))
159
+ if self.single:
160
+ return 1 if answer else 0
161
+ return None
162
+
163
+
164
+ TOOL_ANSWER = AnswerShape()
165
+
166
+
167
+ class CallJournal:
168
+ """A version worker's session's call log on disk beside its store, one JSON line
169
+ per entry as it is kept, so a worker that dies leaves its calls with its rows and
170
+ the next worker resumes both. A call is journaled after its transaction commits:
171
+ a worker that dies in between keeps the call's rows and loses its entry. Once what
172
+ was appended since the last rewrite passes the log's own budget, the file is
173
+ rewritten from the newest of the log's entries that fit ``budget`` bytes: bounded
174
+ on disk whatever the calls carry, and the whole log unless its text is mostly
175
+ escaped in JSON (a control character takes six bytes)."""
176
+
177
+ def __init__(self, path: Path) -> None:
178
+ self.path = Path(path)
179
+ self.appended = 0
180
+
181
+ @staticmethod
182
+ def line(entry: dict) -> bytes:
183
+ return (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
184
+
185
+ def append(self, line: bytes) -> None:
186
+ try:
187
+ self.path.parent.mkdir(parents=True, exist_ok=True)
188
+ with open(self.path, "ab") as journal:
189
+ journal.write(line)
190
+ except OSError as error:
191
+ return self._unwritten(error)
192
+ self.appended += len(line)
193
+
194
+ def rewrite(self, entries: Iterable[dict], budget: int) -> None:
195
+ staging = self.path.with_name(f".{self.path.name}.next")
196
+ try:
197
+ self.path.parent.mkdir(parents=True, exist_ok=True)
198
+ staging.write_bytes(b"".join(self._newest(entries, budget)))
199
+ os.replace(staging, self.path)
200
+ except OSError as error:
201
+ return self._unwritten(error)
202
+ self.appended = 0
203
+
204
+ @classmethod
205
+ def _newest(cls, entries: Iterable[dict], budget: int) -> list[bytes]:
206
+ kept: list[bytes] = []
207
+ for line in map(cls.line, reversed(list(entries))):
208
+ budget -= len(line)
209
+ if budget < 0:
210
+ break
211
+ kept.append(line)
212
+ return kept[::-1]
213
+
214
+ def _unwritten(self, error: OSError) -> None:
215
+ log_line(f"call journal {self.path.name} not written: {error}")
216
+
217
+ @staticmethod
218
+ def read(path: Path) -> list[dict]:
219
+ """The entries a journal holds, oldest first, read by the worker that continues the session
220
+ from it: :meth:`read_left` over the whole file, so a worker killed mid-write or a tenant's
221
+ directory, link or FIFO at the path costs that journal's entries, never the reopen."""
222
+ return CallJournal._read(path, None)
223
+
224
+ @staticmethod
225
+ def read_left(path: Path, max_bytes: int) -> list[dict]:
226
+ """The newest ``max_bytes`` of a journal a dead worker left, read by the supervisor: the
227
+ worker ran tenant code, so a link, a FIFO, a directory or anything but a regular file
228
+ reads as empty, and so does a file that cannot be read."""
229
+ return CallJournal._read(path, max_bytes)
230
+
231
+ @staticmethod
232
+ def _read(path: Path, max_bytes: int | None) -> list[dict]:
233
+ try:
234
+ text = read_tail(path, max_bytes).decode("utf-8", "replace")
235
+ except OSError:
236
+ return []
237
+ return CallJournal._entries(text)
238
+
239
+ @staticmethod
240
+ def _entries(text: str) -> list[dict]:
241
+ """The entries of a journal's text, one to a line. Lines end at ``\\n`` alone: ``str.splitlines``
242
+ also cuts at U+2028, U+2029 and U+0085, which :meth:`line` leaves raw inside an entry."""
243
+ return [entry for entry in map(CallJournal._entry, text.split("\n")) if entry is not None]
244
+
245
+ @staticmethod
246
+ def _entry(line: str) -> dict | None:
247
+ try:
248
+ entry = json.loads(line)
249
+ except (ValueError, RecursionError):
250
+ return None
251
+ return entry if isinstance(entry, dict) and isinstance(entry.get("seq"), int) else None
252
+
253
+
254
+ class CallLog:
255
+ """The calls one session answered, in order: what the agent asked (a
256
+ route's operation, method, path, path args, query and body; a tool's name
257
+ and arguments), the door it came in by (``via``) and what came back
258
+ (status; ``rows``, the rows the answer carried as :class:`AnswerShape`
259
+ reads them; elapsed). ``selectors`` is every string the request carried
260
+ wherever it sat, and ``found`` is "answered with rows", so a grader's
261
+ scalar ``where`` can ask either without the vendor's request shape.
262
+ :class:`CallRedaction` decides what of a request is kept. A request too
263
+ large or too deep to keep names what it lacks in ``withheld``: a route call
264
+ keeps its request line (path, path arguments, query), which HTTP bounds,
265
+ and withholds its body (``["body", "selectors"]``); a tool call withholds its
266
+ arguments. Bounded at ``maximum`` entries (``WORLDS_CALL_LOG_MAX``, default
267
+ :data:`DEFAULT_CALL_LOG_MAX`; ``0`` keeps none) and at what its entries hold,
268
+ ``max_values`` values and ``max_chars`` characters (the sum of their
269
+ :class:`RequestBudget` spends; about 60 MB of memory at most), the oldest
270
+ dropped first; ``total`` counts past the bound, so the first kept ``seq``
271
+ says how many are gone. Every session's ``state`` and grader state carry it
272
+ (:meth:`into_state`), ``serve()`` answers it at ``/__world/calls`` to its admin
273
+ token, a checkpoint carries it (:meth:`resume`), and a task's
274
+ ``verify.py`` is handed it
275
+ as ``calls`` beside the rows, because a read-only investigation leaves no
276
+ row behind and its calls are the only evidence of what it did.
277
+
278
+ ``outbox`` is the session's half of the world request ledger: every call recorded
279
+ here, and every request no route took (:meth:`heard`), goes there with its
280
+ exchange (:class:`Exchange`) whatever this log's own bounds keep."""
281
+
282
+ def __init__(
283
+ self,
284
+ maximum: int | None = None,
285
+ redaction: CallRedaction | None = None,
286
+ max_values: int = DEFAULT_CALL_LOG_VALUES,
287
+ max_chars: int = DEFAULT_CALL_LOG_CHARS,
288
+ journal: CallJournal | None = None,
289
+ outbox: RequestOutbox | None = None,
290
+ ):
291
+ if maximum is None:
292
+ raw = os.environ.get("WORLDS_CALL_LOG_MAX")
293
+ maximum = int(raw) if raw else DEFAULT_CALL_LOG_MAX
294
+ self.maximum = max(0, int(maximum))
295
+ self.max_values = max_values
296
+ self.max_chars = max_chars
297
+ self.redaction = redaction or CallRedaction()
298
+ self.entries: collections.deque[dict] = collections.deque()
299
+ self._costs: collections.deque[tuple[int, int]] = collections.deque()
300
+ self.values = 0
301
+ self.chars = 0
302
+ self.total = 0
303
+ self.journal = journal
304
+ self.outbox = outbox
305
+ self._lock = threading.Lock()
306
+
307
+ @classmethod
308
+ def for_spec(
309
+ cls,
310
+ spec: Spec | None,
311
+ journal: CallJournal | None = None,
312
+ outbox: RequestOutbox | None = None,
313
+ secret: str | None = None,
314
+ ) -> CallLog:
315
+ """The call log of one session of the world ``spec`` declares (None: a world of
316
+ tools alone), whose strings never keep ``secret``, the session's credential."""
317
+ return cls(redaction=CallRedaction.of(spec, secret), journal=journal, outbox=outbox)
318
+
319
+ def record(
320
+ self,
321
+ operation: Operation,
322
+ method: str,
323
+ path: str,
324
+ args: dict,
325
+ query_all: dict,
326
+ body: Any,
327
+ status: int,
328
+ answer: Any,
329
+ elapsed_s: float,
330
+ via: Via | None,
331
+ *,
332
+ shape: AnswerShape,
333
+ served: Served | None = None,
334
+ ) -> None:
335
+ door = {} if via is None else {"via": str(via)}
336
+ heading = {"operation": operation.name, "method": method}
337
+ try:
338
+ request, budget = self._route_request(operation, path, args, query_all, body)
339
+ except (RecursionError, RequestTooLarge):
340
+ request, budget = self._route_line(operation, path, args, query_all)
341
+ answered = _answered(status, shape.rows(answer), elapsed_s)
342
+ secret = served.secret if served else None
343
+ line = {
344
+ **heading,
345
+ "path": CallRedaction.path_without_secret(request["path"], secret),
346
+ "query": CallRedaction.without_secret(request["query"], secret),
347
+ "via": door.get("via"),
348
+ }
349
+ exchange = partial(Exchange.of, served, body, status, answer, self.redaction) if served else None
350
+ self._send(line, answered, exchange)
351
+ if self.maximum > 0:
352
+ self._append({**heading, **request, **door}, answered, budget)
353
+
354
+ def heard(
355
+ self, method: str, path: str, query: str, status: int, answer: Any, via: Via | None, served: Served
356
+ ) -> None:
357
+ """A request no route took (refused, or answered before routing): the ledger keeps it,
358
+ the grader's log does not."""
359
+ try:
360
+ kept_query = self.redaction.query({k: list(v) for k, v in urllib.parse.parse_qs(query).items()})
361
+ except (RecursionError, RequestTooLarge):
362
+ kept_query = {}
363
+ kept_query = CallRedaction.without_secret(kept_query, served.secret)
364
+ line = {"operation": None, "method": method, "path": self.redaction.raw_path(path, served.secret)}
365
+ exchange = partial(Exchange.of, served, None, status, answer, self.redaction)
366
+ door = None if via is None else str(via)
367
+ self._send({**line, "query": kept_query, "via": door}, _answered(status, None, 0.0), exchange)
368
+
369
+ def _send(self, line: dict, answered: dict, exchange: Callable[[], Exchange] | None) -> None:
370
+ if self.outbox is None or exchange is None:
371
+ return
372
+ found = {"rows": answered["rows"], "found": answered["found"], "durationMs": answered["elapsed_ms"]}
373
+ self.outbox.append({**line, **_bounded(line), **found}, exchange())
374
+
375
+ def _route_request(
376
+ self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
377
+ ) -> tuple[dict, RequestBudget]:
378
+ budget = RequestBudget()
379
+ line = self._request_line(operation, path, args, query_all, budget)
380
+ logged_body = self.redaction.value(body, budget) if isinstance(body, (dict, list)) and body else None
381
+ selectors = _strings_in([line["args"], line["query"], logged_body])
382
+ return {**line, "body": logged_body, "selectors": selectors}, budget
383
+
384
+ def _route_line(self, operation: Operation, path: str, args: dict, query_all: dict) -> tuple[dict, RequestBudget]:
385
+ budget = RequestBudget()
386
+ try:
387
+ line = self._request_line(operation, path, args, query_all, budget)
388
+ except (RecursionError, RequestTooLarge):
389
+ return {"path": operation.path, **WITHHELD_ROUTE}, RequestBudget()
390
+ selectors = _strings_in([line["args"], line["query"]])
391
+ return {**line, "body": None, "selectors": selectors, WITHHELD_FIELD: ["body", "selectors"]}, budget
392
+
393
+ def _request_line(
394
+ self, operation: Operation, path: str, args: dict, query_all: dict, budget: RequestBudget
395
+ ) -> dict:
396
+ redaction = self.redaction
397
+ logged_path = redaction.path(operation, path, args)
398
+ budget.spend(1, len(logged_path))
399
+ return {
400
+ "path": logged_path,
401
+ "args": redaction.value(dict(args), budget),
402
+ "query": redaction.query(query_all, budget),
403
+ }
404
+
405
+ def refused_tool(self, tool: str, args: Any, refusal: dict) -> None:
406
+ """A tool call the door refused before it ran (an unknown or disallowed name): the
407
+ ledger keeps it with the refusal it was answered, the grader's log does not."""
408
+ answer = {"error": refusal.get("code"), "message": refusal.get("message")}
409
+ exchange = partial(Exchange.of_tool, args, 400, answer, self.redaction)
410
+ line = {"operation": tool, "method": None, "path": None, "query": {}, "via": str(Via.TOOL)}
411
+ self._send(line, _answered(400, None, 0.0), exchange)
412
+
413
+ def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
414
+ heading = {"operation": tool, "method": None, "path": None}
415
+ answered = _answered(status, TOOL_ANSWER.rows(answer), elapsed_s)
416
+ exchange = partial(Exchange.of_tool, args, status, answer, self.redaction)
417
+ self._send({**heading, "query": {}, "via": str(Via.TOOL)}, answered, exchange)
418
+ if self.maximum == 0:
419
+ return
420
+ try:
421
+ request, budget = self._tool_request(args)
422
+ except (RecursionError, RequestTooLarge):
423
+ request, budget = dict(WITHHELD_TOOL), RequestBudget()
424
+ self._append({**heading, **request, "via": str(Via.TOOL)}, answered, budget)
425
+
426
+ def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
427
+ budget = RequestBudget()
428
+ logged_args = self.redaction.value(args, budget)
429
+ return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
430
+
431
+ def _append(self, request: dict, answered: dict, budget: RequestBudget) -> None:
432
+ at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
433
+ cost = budget.spent()
434
+ with self._lock:
435
+ self.total += 1
436
+ entry = {"seq": self.total, "at": at, **request, **answered}
437
+ self._keep(entry, cost)
438
+ self._journal(entry)
439
+
440
+ def _journal(self, entry: dict) -> None:
441
+ if self.journal is None:
442
+ return
443
+ line = CallJournal.line(entry)
444
+ if self.journal.appended + len(line) > 2 * self.max_chars:
445
+ return self._rewrite_journal()
446
+ self.journal.append(line)
447
+
448
+ def _rewrite_journal(self) -> None:
449
+ if self.journal is not None:
450
+ self.journal.rewrite(self.entries, 4 * self.max_chars)
451
+
452
+ def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
453
+ self.entries.append(entry)
454
+ self._costs.append(cost)
455
+ self.values += cost[0]
456
+ self.chars += cost[1]
457
+ while len(self.entries) > self.maximum or (len(self.entries) > 1 and self._over_budget()):
458
+ self.entries.popleft()
459
+ values, chars = self._costs.popleft()
460
+ self.values -= values
461
+ self.chars -= chars
462
+
463
+ def _over_budget(self) -> bool:
464
+ return self.values > self.max_values or self.chars > self.max_chars
465
+
466
+ def _held(self, call: dict) -> tuple[int, int]:
467
+ budget = RequestBudget()
468
+ try:
469
+ self.redaction.value([call.get(field) for field in ("path", "args", "query", "body")], budget)
470
+ except (RecursionError, RequestTooLarge):
471
+ return MAX_LOGGED_VALUES, MAX_LOGGED_CHARS
472
+ return budget.spent()
473
+
474
+ def describe(self) -> dict:
475
+ with self._lock:
476
+ return {"total": self.total, "kept": len(self.entries), "max": self.maximum, "calls": list(self.entries)}
477
+
478
+ def into_state(self, state: dict) -> dict:
479
+ """``state`` with the calls beside its rows, as every session's ``state`` and grader state carry them."""
480
+ state["calls"] = self.describe()["calls"]
481
+ return state
482
+
483
+ def resume(self, calls: list[dict]) -> None:
484
+ """Carry on from the calls a checkpoint kept: the same entries, numbered on
485
+ from the last of them, each charged what it holds. A log that keeps
486
+ nothing keeps none of them either."""
487
+ if self.maximum == 0:
488
+ return
489
+ costs = [self._held(call) for call in calls]
490
+ with self._lock:
491
+ for call, cost in zip(calls, costs):
492
+ self._keep(call, cost)
493
+ self.total = max([self.total, *(call["seq"] for call in calls)])
494
+ self._rewrite_journal()
495
+
496
+ def clear(self) -> None:
497
+ with self._lock:
498
+ self.entries.clear()
499
+ self._costs.clear()
500
+ self.values = 0
501
+ self.chars = 0
502
+ self.total = 0
503
+ self._rewrite_journal()