@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -0,0 +1,309 @@
1
+ """What the world request ledger keeps of one request a session's world answered: the
2
+ request's headers and each body's first :data:`CAPTURE_BYTES` as text, beside the
3
+ body's size as it went over the wire (``bodyBytes``) and whether the text was cut
4
+ (``truncated``).
5
+
6
+ A JSON body is kept as the call log keeps a request (:class:`CallRedaction`): a
7
+ credential by its name or its shape reads ``[redacted]``, and only the part of the
8
+ body the capture keeps is walked, so a large answer costs no more than its first
9
+ :data:`CAPTURE_BYTES`. A request's body is what the world read (a form's fields
10
+ included), redacted and written as JSON, so its text is not the wire's bytes; a
11
+ body the world did not read (a GET's) is not kept and reads ``truncated``. A body
12
+ answered as bytes is kept as its text when its first :data:`CAPTURE_BYTES` are UTF-8
13
+ and not at all when they are not (``body: null`` beside its size). Every body kept,
14
+ JSON or text, is scrubbed of the secrets it embeds before it is cut
15
+ (:meth:`CallRedaction.scrub`: the session's credential as sent and percent-encoded,
16
+ the value of a query parameter the world's ``[auth]`` carries a secret in, a JSON Web
17
+ Token), read :data:`~.call_redaction.SCRUB_MARGIN` past the cut so a secret across it
18
+ is seen whole; text the capture did not read to its end keeps nothing within that
19
+ margin of where it stopped. Header values are kept only under names that
20
+ say what they are (:meth:`CallRedaction.headers`), and each content type is its
21
+ first :data:`CONTENT_TYPE_CHARS` characters, the request's read from its kept
22
+ header.
23
+
24
+ A tool called by name is an exchange too: its arguments are the request's body and
25
+ its answer the response's, each as JSON. An exchange the request ledger's outbox sheds
26
+ (:class:`~.request_ledger.RequestOutbox`) keeps its sizes and status and gives up its
27
+ headers, text and content types, each body then ``truncated``.
28
+
29
+ The answer as the ledger keeps it is the answer as it goes on the wire
30
+ (:class:`ServedAnswer`), written by every world server (:class:`WorldRequestHandler`); a HEAD is
31
+ kept as the GET it mirrors, whose body the wire leaves off.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import json
37
+ from dataclasses import dataclass, replace
38
+ from http.server import BaseHTTPRequestHandler
39
+ from typing import Any
40
+
41
+ from ..world_session_contract import REQUEST_BODY_CAPTURE_BYTES
42
+ from .call_redaction import SCRUB_MARGIN, CallRedaction
43
+
44
+ CAPTURE_BYTES = REQUEST_BODY_CAPTURE_BYTES
45
+ CONTENT_TYPE_CHARS = 256
46
+ JSON_TYPE = "application/json"
47
+ _MAX_CUT_CHARACTER = 3
48
+
49
+
50
+ def _utf8(head: bytes, cut: bool) -> str | None:
51
+ """``head`` as text, or None when it is not UTF-8. A head cut from a longer body
52
+ may end inside a character, whose bytes are let go; a whole body is text only
53
+ when every byte of it is."""
54
+ for trim in range(_MAX_CUT_CHARACTER + 1 if cut else 1):
55
+ try:
56
+ return head[: len(head) - trim].decode("utf-8")
57
+ except UnicodeDecodeError:
58
+ continue
59
+ return None
60
+
61
+
62
+ def _utf8_lead(head: bytes) -> int:
63
+ """How many leading bytes of ``head`` are UTF-8."""
64
+ try:
65
+ head.decode("utf-8")
66
+ except UnicodeDecodeError as error:
67
+ return error.start
68
+ return len(head)
69
+
70
+
71
+ def _typed(content_type: Any) -> str | None:
72
+ return None if content_type is None else str(content_type)[:CONTENT_TYPE_CHARS]
73
+
74
+
75
+ @dataclass(frozen=True)
76
+ class ServedAnswer:
77
+ """One ``WorldApp`` answer as it goes on the wire: nothing for a 204 or a
78
+ ``None``, bytes as they are, anything else as JSON labelled
79
+ ``application/json`` (an error ``error_content_type``) unless the answer named
80
+ its own type."""
81
+
82
+ headers: dict[str, str]
83
+ payload: bytes
84
+
85
+ @classmethod
86
+ def of(cls, status: int, body: Any, extra: dict[str, str] | None, error_content_type: str) -> ServedAnswer:
87
+ headers = dict(extra or {})
88
+ if status == 204 or body is None:
89
+ return cls(headers, b"")
90
+ if isinstance(body, bytes):
91
+ return cls(headers, body)
92
+ labelled = {"Content-Type": error_content_type if status >= 400 else "application/json", **headers}
93
+ return cls(labelled, json.dumps(body, ensure_ascii=False).encode("utf-8"))
94
+
95
+ @property
96
+ def content_type(self) -> str | None:
97
+ return next((value for name, value in self.headers.items() if name.lower() == "content-type"), None)
98
+
99
+
100
+ def send_response(
101
+ handler: BaseHTTPRequestHandler, status: int, body: Any, extra: dict[str, str] | None, error_content_type: str
102
+ ) -> None:
103
+ """Write one ``WorldApp`` answer to a request handler's socket."""
104
+ answer = ServedAnswer.of(status, body, extra, error_content_type)
105
+ handler.send_response(status)
106
+ for name, value in answer.headers.items():
107
+ handler.send_header(name, value)
108
+ handler.send_header("Content-Length", str(len(answer.payload)))
109
+ handler.end_headers()
110
+ write_body(handler, answer.payload)
111
+
112
+
113
+ def write_body(handler: BaseHTTPRequestHandler, payload: bytes) -> None:
114
+ """An answer's body, left off the wire for a HEAD, whose headers are the GET's."""
115
+ if handler.command != "HEAD":
116
+ handler.wfile.write(payload)
117
+
118
+
119
+ class WorldRequestHandler(BaseHTTPRequestHandler):
120
+ """The methods every world server answers, each through :meth:`_dispatch`: HEAD as the GET
121
+ it mirrors (:func:`write_body` leaves its body off), OPTIONS as the world answers it. A
122
+ request the standard library refuses before any of them (another method, a malformed
123
+ request line) is answered in JSON, never the library's HTML page. No request is
124
+ access-logged: a query-auth world's key rides the request line."""
125
+
126
+ def _dispatch(self, method: str) -> None:
127
+ raise NotImplementedError
128
+
129
+ def log_message(self, format: str, *args: object) -> None:
130
+ pass
131
+
132
+ def send_error(self, code: int, message: str | None = None, explain: str | None = None) -> None:
133
+ self.close_connection = True
134
+ reason = self.responses.get(code, ("error",))[0]
135
+ send_response(self, code, {"error": reason}, {"Connection": "close"}, JSON_TYPE)
136
+
137
+ def do_GET(self) -> None:
138
+ self._dispatch("GET")
139
+
140
+ def do_HEAD(self) -> None:
141
+ self._dispatch("GET")
142
+
143
+ def do_POST(self) -> None:
144
+ self._dispatch("POST")
145
+
146
+ def do_PUT(self) -> None:
147
+ self._dispatch("PUT")
148
+
149
+ def do_PATCH(self) -> None:
150
+ self._dispatch("PATCH")
151
+
152
+ def do_DELETE(self) -> None:
153
+ self._dispatch("DELETE")
154
+
155
+ def do_OPTIONS(self) -> None:
156
+ self._dispatch("OPTIONS")
157
+
158
+
159
+ @dataclass(frozen=True)
160
+ class Served:
161
+ """What the request ledger reads of one route request beyond what the call log
162
+ keeps: the headers the world read, the body's size as it arrived, the
163
+ answer's own headers, how the world labels an error, and the session's token,
164
+ which no kept header may carry."""
165
+
166
+ headers: dict[str, str]
167
+ body_bytes: int
168
+ extra: dict[str, str] | None
169
+ error_content_type: str
170
+ secret: str | None
171
+ read_body: bool = True
172
+
173
+
174
+ @dataclass(frozen=True)
175
+ class CapturedBody:
176
+ """One body as the request ledger keeps it (module docstring); ``size`` is None
177
+ for a value that could not be written as JSON."""
178
+
179
+ content_type: str | None
180
+ size: int | None
181
+ text: str | None = None
182
+ truncated: bool = False
183
+
184
+ @classmethod
185
+ def of_request(cls, body: Any, served: Served, content_type: Any, redaction: CallRedaction) -> CapturedBody:
186
+ typed, size = _typed(content_type), served.body_bytes
187
+ if size == 0:
188
+ return cls(typed, 0)
189
+ if not served.read_body:
190
+ return cls(typed, size, None, True)
191
+ return cls.of_value(body, size, typed, redaction)
192
+
193
+ @classmethod
194
+ def of_answer(cls, status: int, answer: Any, served: Served, redaction: CallRedaction) -> CapturedBody:
195
+ try:
196
+ wire = ServedAnswer.of(status, answer, served.extra, served.error_content_type)
197
+ except (RecursionError, TypeError, ValueError):
198
+ return cls(None, None, None, True)
199
+ size, content_type = len(wire.payload), _typed(wire.content_type)
200
+ if size == 0:
201
+ return cls(content_type, 0)
202
+ if isinstance(answer, bytes):
203
+ return cls.of_bytes(wire.payload, content_type, redaction)
204
+ return cls.of_value(answer, size, content_type, redaction)
205
+
206
+ @classmethod
207
+ def of_document(cls, value: Any, redaction: CallRedaction) -> CapturedBody:
208
+ """A tool's arguments or answer: the JSON it is, its size as JSON."""
209
+ if value is None:
210
+ return cls(None, 0)
211
+ try:
212
+ size = len(json.dumps(value, ensure_ascii=False).encode("utf-8", "replace"))
213
+ except (RecursionError, TypeError, ValueError):
214
+ return cls(JSON_TYPE, None, None, True)
215
+ return cls.of_value(value, size, JSON_TYPE, redaction)
216
+
217
+ @classmethod
218
+ def of_bytes(cls, payload: bytes, content_type: str | None, redaction: CallRedaction) -> CapturedBody:
219
+ """A body answered as bytes (module docstring): the UTF-8 text of its first
220
+ :data:`CAPTURE_BYTES` and :data:`SCRUB_MARGIN` more, scrubbed, then cut."""
221
+ head = payload[: CAPTURE_BYTES + SCRUB_MARGIN]
222
+ lead = _utf8_lead(head)
223
+ whole = lead == len(payload)
224
+ if lead < (len(payload) if len(payload) <= CAPTURE_BYTES else CAPTURE_BYTES - _MAX_CUT_CHARACTER):
225
+ return cls(_typed(content_type), len(payload))
226
+ scrubbed = redaction.scrub(head[:lead].decode("utf-8")).encode("utf-8")
227
+ limit = CAPTURE_BYTES if whole else max(0, min(CAPTURE_BYTES, len(scrubbed) - SCRUB_MARGIN))
228
+ cut = not whole or len(scrubbed) > CAPTURE_BYTES
229
+ return cls(_typed(content_type), len(payload), _utf8(scrubbed[:limit], cut), cut)
230
+
231
+ @classmethod
232
+ def of_value(cls, value: Any, size: int, content_type: str | None, redaction: CallRedaction) -> CapturedBody:
233
+ try:
234
+ kept, cut = redaction.prefix(value, CAPTURE_BYTES)
235
+ text = redaction.scrub(json.dumps(kept, ensure_ascii=False))
236
+ except (RecursionError, TypeError, ValueError):
237
+ return cls(content_type, size, None, True)
238
+ encoded = text.encode("utf-8", "replace")
239
+ longer = len(encoded) > CAPTURE_BYTES
240
+ return cls(content_type, size, _utf8(encoded[:CAPTURE_BYTES], longer), cut or longer)
241
+
242
+ @classmethod
243
+ def from_json(cls, data: dict) -> CapturedBody:
244
+ return cls(data.get("contentType"), data.get("bodyBytes"), data.get("body"), bool(data.get("truncated")))
245
+
246
+ def shed(self) -> CapturedBody:
247
+ return replace(self, text=None, content_type=None, truncated=self.truncated or self.text is not None)
248
+
249
+ @property
250
+ def charge(self) -> int:
251
+ return len(self.text or "") + len(self.content_type or "")
252
+
253
+ def as_json(self) -> dict:
254
+ return {
255
+ "body": self.text,
256
+ "bodyBytes": self.size,
257
+ "truncated": self.truncated,
258
+ "contentType": self.content_type,
259
+ }
260
+
261
+
262
+ @dataclass(frozen=True)
263
+ class Exchange:
264
+ """One HTTP request a session's world answered and the answer it gave, as
265
+ the request ledger keeps them."""
266
+
267
+ headers: dict[str, str]
268
+ request: CapturedBody
269
+ status: int
270
+ response: CapturedBody
271
+
272
+ @classmethod
273
+ def of(cls, served: Served, body: Any, status: int, answer: Any, redaction: CallRedaction) -> Exchange:
274
+ headers = redaction.headers(served.headers, served.secret, CAPTURE_BYTES)
275
+ return cls(
276
+ headers,
277
+ CapturedBody.of_request(body, served, headers.get("content-type"), redaction),
278
+ status,
279
+ CapturedBody.of_answer(status, answer, served, redaction),
280
+ )
281
+
282
+ @classmethod
283
+ def of_tool(cls, args: Any, status: int, answer: Any, redaction: CallRedaction) -> Exchange:
284
+ """A tool called by name: its arguments are what was asked, its answer what came back."""
285
+ return cls({}, CapturedBody.of_document(args, redaction), status, CapturedBody.of_document(answer, redaction))
286
+
287
+ @classmethod
288
+ def from_json(cls, data: dict) -> Exchange:
289
+ request, response = data.get("request") or {}, data.get("response") or {}
290
+ return cls(
291
+ dict(request.get("headers") or {}),
292
+ CapturedBody.from_json(request),
293
+ int(response.get("status") or 0),
294
+ CapturedBody.from_json(response),
295
+ )
296
+
297
+ def shed(self) -> Exchange:
298
+ return Exchange({}, self.request.shed(), self.status, self.response.shed())
299
+
300
+ @property
301
+ def charge(self) -> int:
302
+ held = sum(len(name) + len(str(value)) for name, value in self.headers.items())
303
+ return held + self.request.charge + self.response.charge
304
+
305
+ def as_json(self) -> dict:
306
+ return {
307
+ "request": {"headers": self.headers, **self.request.as_json()},
308
+ "response": {"status": self.status, **self.response.as_json()},
309
+ }
@@ -3,17 +3,31 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import collections
6
+ import json
6
7
  import os
8
+ import stat
7
9
  import threading
10
+ import urllib.parse
8
11
  from dataclasses import dataclass
9
12
  from datetime import datetime, timezone
10
13
  from enum import StrEnum
11
- from typing import Any
14
+ from functools import partial
15
+ from pathlib import Path
16
+ from typing import TYPE_CHECKING, Any, Callable, Iterable
12
17
 
18
+ from .call_capture import Exchange, Served
13
19
  from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
14
20
  from .connector import Operation, Spec, _dig
21
+ from .handler_errors import log_line
15
22
  from .tasks import WITHHELD_FIELD
16
23
 
24
+ if TYPE_CHECKING:
25
+ from .request_ledger import RequestOutbox
26
+
27
+ METHOD_CHARS = 32
28
+ OPERATION_CHARS = 512
29
+ PATH_CHARS = 64 << 10
30
+ LINE_BOUNDS = {"method": METHOD_CHARS, "operation": OPERATION_CHARS, "path": PATH_CHARS}
17
31
  DEFAULT_CALL_LOG_MAX = 5000
18
32
  DEFAULT_CALL_LOG_VALUES = 200_000
19
33
  DEFAULT_CALL_LOG_CHARS = 8 << 20
@@ -27,6 +41,35 @@ WITHHELD_ROUTE = {
27
41
  WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
28
42
 
29
43
 
44
+ def read_tail(path: str | Path, max_bytes: int | None, dir_fd: int | None = None) -> bytes:
45
+ """The newest ``max_bytes`` of the file at ``path`` (all of it when None; relative to ``dir_fd`` when
46
+ given) as tenant code could have left it: no link followed, no FIFO waited on, anything but a regular
47
+ file read as empty. Raises OSError when it cannot be opened or read; its descriptor is closed on
48
+ every path."""
49
+ descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dir_fd)
50
+ try:
51
+ found = os.fstat(descriptor)
52
+ if not stat.S_ISREG(found.st_mode):
53
+ return b""
54
+ size = found.st_size if max_bytes is None else min(max_bytes, found.st_size)
55
+ os.lseek(descriptor, found.st_size - size, os.SEEK_SET)
56
+ return _read_up_to(descriptor, size)
57
+ finally:
58
+ os.close(descriptor)
59
+
60
+
61
+ def _read_up_to(descriptor: int, max_bytes: int) -> bytes:
62
+ chunks: list[bytes] = []
63
+ left = max_bytes
64
+ while left > 0:
65
+ chunk = os.read(descriptor, min(left, 1 << 20))
66
+ if not chunk:
67
+ break
68
+ chunks.append(chunk)
69
+ left -= len(chunk)
70
+ return b"".join(chunks)
71
+
72
+
30
73
  def _children(node: Any) -> Any:
31
74
  if isinstance(node, dict):
32
75
  return node.values()
@@ -63,6 +106,21 @@ def _found(status: int, rows: int | None) -> bool | None:
63
106
  return None if rows is None else rows > 0
64
107
 
65
108
 
109
+ def _bounded(line: dict) -> dict:
110
+ """A ledger line's free-text fields cut to the ledger's bounds (``limits`` in
111
+ conformance/world-request-log.json), so the platform keeps the request."""
112
+ return {
113
+ field: value[:size] if isinstance(value, str) else value
114
+ for field, size in LINE_BOUNDS.items()
115
+ for value in [line.get(field)]
116
+ }
117
+
118
+
119
+ def _answered(status: int, rows: int | None, elapsed_s: float) -> dict:
120
+ rows = None if status >= 400 else rows
121
+ return {"status": status, "rows": rows, "found": _found(status, rows), "elapsed_ms": round(elapsed_s * 1000, 1)}
122
+
123
+
66
124
  class Via(StrEnum):
67
125
  """The door a call came in by, set by the server from where it arrived and
68
126
  never from a request header. It names the door, not the actor: a token
@@ -106,6 +164,93 @@ class AnswerShape:
106
164
  TOOL_ANSWER = AnswerShape()
107
165
 
108
166
 
167
+ class CallJournal:
168
+ """A version worker's session's call log on disk beside its store, one JSON line
169
+ per entry as it is kept, so a worker that dies leaves its calls with its rows and
170
+ the next worker resumes both. A call is journaled after its transaction commits:
171
+ a worker that dies in between keeps the call's rows and loses its entry. Once what
172
+ was appended since the last rewrite passes the log's own budget, the file is
173
+ rewritten from the newest of the log's entries that fit ``budget`` bytes: bounded
174
+ on disk whatever the calls carry, and the whole log unless its text is mostly
175
+ escaped in JSON (a control character takes six bytes)."""
176
+
177
+ def __init__(self, path: Path) -> None:
178
+ self.path = Path(path)
179
+ self.appended = 0
180
+
181
+ @staticmethod
182
+ def line(entry: dict) -> bytes:
183
+ return (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
184
+
185
+ def append(self, line: bytes) -> None:
186
+ try:
187
+ self.path.parent.mkdir(parents=True, exist_ok=True)
188
+ with open(self.path, "ab") as journal:
189
+ journal.write(line)
190
+ except OSError as error:
191
+ return self._unwritten(error)
192
+ self.appended += len(line)
193
+
194
+ def rewrite(self, entries: Iterable[dict], budget: int) -> None:
195
+ staging = self.path.with_name(f".{self.path.name}.next")
196
+ try:
197
+ self.path.parent.mkdir(parents=True, exist_ok=True)
198
+ staging.write_bytes(b"".join(self._newest(entries, budget)))
199
+ os.replace(staging, self.path)
200
+ except OSError as error:
201
+ return self._unwritten(error)
202
+ self.appended = 0
203
+
204
+ @classmethod
205
+ def _newest(cls, entries: Iterable[dict], budget: int) -> list[bytes]:
206
+ kept: list[bytes] = []
207
+ for line in map(cls.line, reversed(list(entries))):
208
+ budget -= len(line)
209
+ if budget < 0:
210
+ break
211
+ kept.append(line)
212
+ return kept[::-1]
213
+
214
+ def _unwritten(self, error: OSError) -> None:
215
+ log_line(f"call journal {self.path.name} not written: {error}")
216
+
217
+ @staticmethod
218
+ def read(path: Path) -> list[dict]:
219
+ """The entries a journal holds, oldest first, read by the worker that continues the session
220
+ from it: :meth:`read_left` over the whole file, so a worker killed mid-write or a tenant's
221
+ directory, link or FIFO at the path costs that journal's entries, never the reopen."""
222
+ return CallJournal._read(path, None)
223
+
224
+ @staticmethod
225
+ def read_left(path: Path, max_bytes: int) -> list[dict]:
226
+ """The newest ``max_bytes`` of a journal a dead worker left, read by the supervisor: the
227
+ worker ran tenant code, so a link, a FIFO, a directory or anything but a regular file
228
+ reads as empty, and so does a file that cannot be read."""
229
+ return CallJournal._read(path, max_bytes)
230
+
231
+ @staticmethod
232
+ def _read(path: Path, max_bytes: int | None) -> list[dict]:
233
+ try:
234
+ text = read_tail(path, max_bytes).decode("utf-8", "replace")
235
+ except OSError:
236
+ return []
237
+ return CallJournal._entries(text)
238
+
239
+ @staticmethod
240
+ def _entries(text: str) -> list[dict]:
241
+ """The entries of a journal's text, one to a line. Lines end at ``\\n`` alone: ``str.splitlines``
242
+ also cuts at U+2028, U+2029 and U+0085, which :meth:`line` leaves raw inside an entry."""
243
+ return [entry for entry in map(CallJournal._entry, text.split("\n")) if entry is not None]
244
+
245
+ @staticmethod
246
+ def _entry(line: str) -> dict | None:
247
+ try:
248
+ entry = json.loads(line)
249
+ except (ValueError, RecursionError):
250
+ return None
251
+ return entry if isinstance(entry, dict) and isinstance(entry.get("seq"), int) else None
252
+
253
+
109
254
  class CallLog:
110
255
  """The calls one session answered, in order: what the agent asked (a
111
256
  route's operation, method, path, path args, query and body; a tool's name
@@ -128,7 +273,11 @@ class CallLog:
128
273
  token, a checkpoint carries it (:meth:`resume`), and a task's
129
274
  ``verify.py`` is handed it
130
275
  as ``calls`` beside the rows, because a read-only investigation leaves no
131
- row behind and its calls are the only evidence of what it did."""
276
+ row behind and its calls are the only evidence of what it did.
277
+
278
+ ``outbox`` is the session's half of the world request ledger: every call recorded
279
+ here, and every request no route took (:meth:`heard`), goes there with its
280
+ exchange (:class:`Exchange`) whatever this log's own bounds keep."""
132
281
 
133
282
  def __init__(
134
283
  self,
@@ -136,6 +285,8 @@ class CallLog:
136
285
  redaction: CallRedaction | None = None,
137
286
  max_values: int = DEFAULT_CALL_LOG_VALUES,
138
287
  max_chars: int = DEFAULT_CALL_LOG_CHARS,
288
+ journal: CallJournal | None = None,
289
+ outbox: RequestOutbox | None = None,
139
290
  ):
140
291
  if maximum is None:
141
292
  raw = os.environ.get("WORLDS_CALL_LOG_MAX")
@@ -149,11 +300,21 @@ class CallLog:
149
300
  self.values = 0
150
301
  self.chars = 0
151
302
  self.total = 0
303
+ self.journal = journal
304
+ self.outbox = outbox
152
305
  self._lock = threading.Lock()
153
306
 
154
307
  @classmethod
155
- def for_spec(cls, spec: Spec) -> CallLog:
156
- return cls(redaction=CallRedaction.of(spec))
308
+ def for_spec(
309
+ cls,
310
+ spec: Spec | None,
311
+ journal: CallJournal | None = None,
312
+ outbox: RequestOutbox | None = None,
313
+ secret: str | None = None,
314
+ ) -> CallLog:
315
+ """The call log of one session of the world ``spec`` declares (None: a world of
316
+ tools alone), whose strings never keep ``secret``, the session's credential."""
317
+ return cls(redaction=CallRedaction.of(spec, secret), journal=journal, outbox=outbox)
157
318
 
158
319
  def record(
159
320
  self,
@@ -169,16 +330,47 @@ class CallLog:
169
330
  via: Via | None,
170
331
  *,
171
332
  shape: AnswerShape,
333
+ served: Served | None = None,
172
334
  ) -> None:
173
- if self.maximum == 0:
174
- return
175
335
  door = {} if via is None else {"via": str(via)}
176
336
  heading = {"operation": operation.name, "method": method}
177
337
  try:
178
338
  request, budget = self._route_request(operation, path, args, query_all, body)
179
339
  except (RecursionError, RequestTooLarge):
180
340
  request, budget = self._route_line(operation, path, args, query_all)
181
- self._append({**heading, **request, **door}, status, shape.rows(answer), elapsed_s, budget)
341
+ answered = _answered(status, shape.rows(answer), elapsed_s)
342
+ secret = served.secret if served else None
343
+ line = {
344
+ **heading,
345
+ "path": CallRedaction.path_without_secret(request["path"], secret),
346
+ "query": CallRedaction.without_secret(request["query"], secret),
347
+ "via": door.get("via"),
348
+ }
349
+ exchange = partial(Exchange.of, served, body, status, answer, self.redaction) if served else None
350
+ self._send(line, answered, exchange)
351
+ if self.maximum > 0:
352
+ self._append({**heading, **request, **door}, answered, budget)
353
+
354
+ def heard(
355
+ self, method: str, path: str, query: str, status: int, answer: Any, via: Via | None, served: Served
356
+ ) -> None:
357
+ """A request no route took (refused, or answered before routing): the ledger keeps it,
358
+ the grader's log does not."""
359
+ try:
360
+ kept_query = self.redaction.query({k: list(v) for k, v in urllib.parse.parse_qs(query).items()})
361
+ except (RecursionError, RequestTooLarge):
362
+ kept_query = {}
363
+ kept_query = CallRedaction.without_secret(kept_query, served.secret)
364
+ line = {"operation": None, "method": method, "path": self.redaction.raw_path(path, served.secret)}
365
+ exchange = partial(Exchange.of, served, None, status, answer, self.redaction)
366
+ door = None if via is None else str(via)
367
+ self._send({**line, "query": kept_query, "via": door}, _answered(status, None, 0.0), exchange)
368
+
369
+ def _send(self, line: dict, answered: dict, exchange: Callable[[], Exchange] | None) -> None:
370
+ if self.outbox is None or exchange is None:
371
+ return
372
+ found = {"rows": answered["rows"], "found": answered["found"], "durationMs": answered["elapsed_ms"]}
373
+ self.outbox.append({**line, **_bounded(line), **found}, exchange())
182
374
 
183
375
  def _route_request(
184
376
  self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
@@ -210,34 +402,52 @@ class CallLog:
210
402
  "query": redaction.query(query_all, budget),
211
403
  }
212
404
 
405
+ def refused_tool(self, tool: str, args: Any, refusal: dict) -> None:
406
+ """A tool call the door refused before it ran (an unknown or disallowed name): the
407
+ ledger keeps it with the refusal it was answered, the grader's log does not."""
408
+ answer = {"error": refusal.get("code"), "message": refusal.get("message")}
409
+ exchange = partial(Exchange.of_tool, args, 400, answer, self.redaction)
410
+ line = {"operation": tool, "method": None, "path": None, "query": {}, "via": str(Via.TOOL)}
411
+ self._send(line, _answered(400, None, 0.0), exchange)
412
+
213
413
  def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
414
+ heading = {"operation": tool, "method": None, "path": None}
415
+ answered = _answered(status, TOOL_ANSWER.rows(answer), elapsed_s)
416
+ exchange = partial(Exchange.of_tool, args, status, answer, self.redaction)
417
+ self._send({**heading, "query": {}, "via": str(Via.TOOL)}, answered, exchange)
214
418
  if self.maximum == 0:
215
419
  return
216
- heading = {"operation": tool, "method": None, "path": None}
217
420
  try:
218
421
  request, budget = self._tool_request(args)
219
422
  except (RecursionError, RequestTooLarge):
220
423
  request, budget = dict(WITHHELD_TOOL), RequestBudget()
221
- self._append({**heading, **request, "via": str(Via.TOOL)}, status, TOOL_ANSWER.rows(answer), elapsed_s, budget)
424
+ self._append({**heading, **request, "via": str(Via.TOOL)}, answered, budget)
222
425
 
223
426
  def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
224
427
  budget = RequestBudget()
225
428
  logged_args = self.redaction.value(args, budget)
226
429
  return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
227
430
 
228
- def _append(self, request: dict, status: int, rows: int | None, elapsed_s: float, budget: RequestBudget) -> None:
229
- rows = None if status >= 400 else rows
230
- answered = {
231
- "status": status,
232
- "rows": rows,
233
- "found": _found(status, rows),
234
- "elapsed_ms": round(elapsed_s * 1000, 1),
235
- }
431
+ def _append(self, request: dict, answered: dict, budget: RequestBudget) -> None:
236
432
  at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
237
433
  cost = budget.spent()
238
434
  with self._lock:
239
435
  self.total += 1
240
- self._keep({"seq": self.total, "at": at, **request, **answered}, cost)
436
+ entry = {"seq": self.total, "at": at, **request, **answered}
437
+ self._keep(entry, cost)
438
+ self._journal(entry)
439
+
440
+ def _journal(self, entry: dict) -> None:
441
+ if self.journal is None:
442
+ return
443
+ line = CallJournal.line(entry)
444
+ if self.journal.appended + len(line) > 2 * self.max_chars:
445
+ return self._rewrite_journal()
446
+ self.journal.append(line)
447
+
448
+ def _rewrite_journal(self) -> None:
449
+ if self.journal is not None:
450
+ self.journal.rewrite(self.entries, 4 * self.max_chars)
241
451
 
242
452
  def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
243
453
  self.entries.append(entry)
@@ -281,6 +491,7 @@ class CallLog:
281
491
  for call, cost in zip(calls, costs):
282
492
  self._keep(call, cost)
283
493
  self.total = max([self.total, *(call["seq"] for call in calls)])
494
+ self._rewrite_journal()
284
495
 
285
496
  def clear(self) -> None:
286
497
  with self._lock:
@@ -289,3 +500,4 @@ class CallLog:
289
500
  self.values = 0
290
501
  self.chars = 0
291
502
  self.total = 0
503
+ self._rewrite_journal()