@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -3,17 +3,34 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import collections
|
|
6
|
+
import json
|
|
6
7
|
import os
|
|
8
|
+
import stat
|
|
7
9
|
import threading
|
|
10
|
+
import urllib.parse
|
|
8
11
|
from dataclasses import dataclass
|
|
9
12
|
from datetime import datetime, timezone
|
|
10
13
|
from enum import StrEnum
|
|
11
|
-
from
|
|
14
|
+
from functools import partial
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Callable, Iterable
|
|
12
17
|
|
|
18
|
+
from .call_capture import Exchange, Served
|
|
13
19
|
from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
|
|
14
20
|
from .connector import Operation, Spec, _dig
|
|
21
|
+
from .handler_errors import log_line
|
|
15
22
|
from .tasks import WITHHELD_FIELD
|
|
16
23
|
|
|
24
|
+
from .webhooks import ANSWER_SUFFIX, SUBSCRIPTIONS_SHOWN, WebhookAnswer, WebhookFeed, WebhookPoll
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from .webhooks import Delivery, WebhookOutbox
|
|
28
|
+
from .request_ledger import RequestOutbox
|
|
29
|
+
|
|
30
|
+
METHOD_CHARS = 32
|
|
31
|
+
OPERATION_CHARS = 512
|
|
32
|
+
PATH_CHARS = 64 << 10
|
|
33
|
+
LINE_BOUNDS = {"method": METHOD_CHARS, "operation": OPERATION_CHARS, "path": PATH_CHARS}
|
|
17
34
|
DEFAULT_CALL_LOG_MAX = 5000
|
|
18
35
|
DEFAULT_CALL_LOG_VALUES = 200_000
|
|
19
36
|
DEFAULT_CALL_LOG_CHARS = 8 << 20
|
|
@@ -27,6 +44,35 @@ WITHHELD_ROUTE = {
|
|
|
27
44
|
WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
|
|
28
45
|
|
|
29
46
|
|
|
47
|
+
def read_tail(path: str | Path, max_bytes: int | None, dir_fd: int | None = None) -> bytes:
|
|
48
|
+
"""The newest ``max_bytes`` of the file at ``path`` (all of it when None; relative to ``dir_fd`` when
|
|
49
|
+
given) as tenant code could have left it: no link followed, no FIFO waited on, anything but a regular
|
|
50
|
+
file read as empty. Raises OSError when it cannot be opened or read; its descriptor is closed on
|
|
51
|
+
every path."""
|
|
52
|
+
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dir_fd)
|
|
53
|
+
try:
|
|
54
|
+
found = os.fstat(descriptor)
|
|
55
|
+
if not stat.S_ISREG(found.st_mode):
|
|
56
|
+
return b""
|
|
57
|
+
size = found.st_size if max_bytes is None else min(max_bytes, found.st_size)
|
|
58
|
+
os.lseek(descriptor, found.st_size - size, os.SEEK_SET)
|
|
59
|
+
return _read_up_to(descriptor, size)
|
|
60
|
+
finally:
|
|
61
|
+
os.close(descriptor)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _read_up_to(descriptor: int, max_bytes: int) -> bytes:
|
|
65
|
+
chunks: list[bytes] = []
|
|
66
|
+
left = max_bytes
|
|
67
|
+
while left > 0:
|
|
68
|
+
chunk = os.read(descriptor, min(left, 1 << 20))
|
|
69
|
+
if not chunk:
|
|
70
|
+
break
|
|
71
|
+
chunks.append(chunk)
|
|
72
|
+
left -= len(chunk)
|
|
73
|
+
return b"".join(chunks)
|
|
74
|
+
|
|
75
|
+
|
|
30
76
|
def _children(node: Any) -> Any:
|
|
31
77
|
if isinstance(node, dict):
|
|
32
78
|
return node.values()
|
|
@@ -63,6 +109,21 @@ def _found(status: int, rows: int | None) -> bool | None:
|
|
|
63
109
|
return None if rows is None else rows > 0
|
|
64
110
|
|
|
65
111
|
|
|
112
|
+
def _bounded(line: dict) -> dict:
|
|
113
|
+
"""A ledger line's free-text fields cut to the ledger's bounds (``limits`` in
|
|
114
|
+
conformance/world-request-log.json), so the platform keeps the request."""
|
|
115
|
+
return {
|
|
116
|
+
field: value[:size] if isinstance(value, str) else value
|
|
117
|
+
for field, size in LINE_BOUNDS.items()
|
|
118
|
+
for value in [line.get(field)]
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _answered(status: int, rows: int | None, elapsed_s: float) -> dict:
|
|
123
|
+
rows = None if status >= 400 else rows
|
|
124
|
+
return {"status": status, "rows": rows, "found": _found(status, rows), "elapsed_ms": round(elapsed_s * 1000, 1)}
|
|
125
|
+
|
|
126
|
+
|
|
66
127
|
class Via(StrEnum):
|
|
67
128
|
"""The door a call came in by, set by the server from where it arrived and
|
|
68
129
|
never from a request header. It names the door, not the actor: a token
|
|
@@ -71,6 +132,7 @@ class Via(StrEnum):
|
|
|
71
132
|
API = "api"
|
|
72
133
|
UI = "ui"
|
|
73
134
|
TOOL = "tool"
|
|
135
|
+
WEBHOOK = "webhook"
|
|
74
136
|
|
|
75
137
|
|
|
76
138
|
@dataclass(frozen=True)
|
|
@@ -106,6 +168,93 @@ class AnswerShape:
|
|
|
106
168
|
TOOL_ANSWER = AnswerShape()
|
|
107
169
|
|
|
108
170
|
|
|
171
|
+
class CallJournal:
|
|
172
|
+
"""A version worker's session's call log on disk beside its store, one JSON line
|
|
173
|
+
per entry as it is kept, so a worker that dies leaves its calls with its rows and
|
|
174
|
+
the next worker resumes both. A call is journaled after its transaction commits:
|
|
175
|
+
a worker that dies in between keeps the call's rows and loses its entry. Once what
|
|
176
|
+
was appended since the last rewrite passes the log's own budget, the file is
|
|
177
|
+
rewritten from the newest of the log's entries that fit ``budget`` bytes: bounded
|
|
178
|
+
on disk whatever the calls carry, and the whole log unless its text is mostly
|
|
179
|
+
escaped in JSON (a control character takes six bytes)."""
|
|
180
|
+
|
|
181
|
+
def __init__(self, path: Path) -> None:
|
|
182
|
+
self.path = Path(path)
|
|
183
|
+
self.appended = 0
|
|
184
|
+
|
|
185
|
+
@staticmethod
|
|
186
|
+
def line(entry: dict) -> bytes:
|
|
187
|
+
return (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
|
|
188
|
+
|
|
189
|
+
def append(self, line: bytes) -> None:
|
|
190
|
+
try:
|
|
191
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
with open(self.path, "ab") as journal:
|
|
193
|
+
journal.write(line)
|
|
194
|
+
except OSError as error:
|
|
195
|
+
return self._unwritten(error)
|
|
196
|
+
self.appended += len(line)
|
|
197
|
+
|
|
198
|
+
def rewrite(self, entries: Iterable[dict], budget: int) -> None:
|
|
199
|
+
staging = self.path.with_name(f".{self.path.name}.next")
|
|
200
|
+
try:
|
|
201
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
202
|
+
staging.write_bytes(b"".join(self._newest(entries, budget)))
|
|
203
|
+
os.replace(staging, self.path)
|
|
204
|
+
except OSError as error:
|
|
205
|
+
return self._unwritten(error)
|
|
206
|
+
self.appended = 0
|
|
207
|
+
|
|
208
|
+
@classmethod
|
|
209
|
+
def _newest(cls, entries: Iterable[dict], budget: int) -> list[bytes]:
|
|
210
|
+
kept: list[bytes] = []
|
|
211
|
+
for line in map(cls.line, reversed(list(entries))):
|
|
212
|
+
budget -= len(line)
|
|
213
|
+
if budget < 0:
|
|
214
|
+
break
|
|
215
|
+
kept.append(line)
|
|
216
|
+
return kept[::-1]
|
|
217
|
+
|
|
218
|
+
def _unwritten(self, error: OSError) -> None:
|
|
219
|
+
log_line(f"call journal {self.path.name} not written: {error}")
|
|
220
|
+
|
|
221
|
+
@staticmethod
|
|
222
|
+
def read(path: Path) -> list[dict]:
|
|
223
|
+
"""The entries a journal holds, oldest first, read by the worker that continues the session
|
|
224
|
+
from it: :meth:`read_left` over the whole file, so a worker killed mid-write or a tenant's
|
|
225
|
+
directory, link or FIFO at the path costs that journal's entries, never the reopen."""
|
|
226
|
+
return CallJournal._read(path, None)
|
|
227
|
+
|
|
228
|
+
@staticmethod
|
|
229
|
+
def read_left(path: Path, max_bytes: int) -> list[dict]:
|
|
230
|
+
"""The newest ``max_bytes`` of a journal a dead worker left, read by the supervisor: the
|
|
231
|
+
worker ran tenant code, so a link, a FIFO, a directory or anything but a regular file
|
|
232
|
+
reads as empty, and so does a file that cannot be read."""
|
|
233
|
+
return CallJournal._read(path, max_bytes)
|
|
234
|
+
|
|
235
|
+
@staticmethod
|
|
236
|
+
def _read(path: Path, max_bytes: int | None) -> list[dict]:
|
|
237
|
+
try:
|
|
238
|
+
text = read_tail(path, max_bytes).decode("utf-8", "replace")
|
|
239
|
+
except OSError:
|
|
240
|
+
return []
|
|
241
|
+
return CallJournal._entries(text)
|
|
242
|
+
|
|
243
|
+
@staticmethod
|
|
244
|
+
def _entries(text: str) -> list[dict]:
|
|
245
|
+
"""The entries of a journal's text, one to a line. Lines end at ``\\n`` alone: ``str.splitlines``
|
|
246
|
+
also cuts at U+2028, U+2029 and U+0085, which :meth:`line` leaves raw inside an entry."""
|
|
247
|
+
return [entry for entry in map(CallJournal._entry, text.split("\n")) if entry is not None]
|
|
248
|
+
|
|
249
|
+
@staticmethod
|
|
250
|
+
def _entry(line: str) -> dict | None:
|
|
251
|
+
try:
|
|
252
|
+
entry = json.loads(line)
|
|
253
|
+
except (ValueError, RecursionError):
|
|
254
|
+
return None
|
|
255
|
+
return entry if isinstance(entry, dict) and isinstance(entry.get("seq"), int) else None
|
|
256
|
+
|
|
257
|
+
|
|
109
258
|
class CallLog:
|
|
110
259
|
"""The calls one session answered, in order: what the agent asked (a
|
|
111
260
|
route's operation, method, path, path args, query and body; a tool's name
|
|
@@ -128,7 +277,11 @@ class CallLog:
|
|
|
128
277
|
token, a checkpoint carries it (:meth:`resume`), and a task's
|
|
129
278
|
``verify.py`` is handed it
|
|
130
279
|
as ``calls`` beside the rows, because a read-only investigation leaves no
|
|
131
|
-
row behind and its calls are the only evidence of what it did.
|
|
280
|
+
row behind and its calls are the only evidence of what it did.
|
|
281
|
+
|
|
282
|
+
``outbox`` is the session's half of the world request ledger: every call recorded
|
|
283
|
+
here, and every request no route took (:meth:`heard`), goes there with its
|
|
284
|
+
exchange (:class:`Exchange`) whatever this log's own bounds keep."""
|
|
132
285
|
|
|
133
286
|
def __init__(
|
|
134
287
|
self,
|
|
@@ -136,6 +289,8 @@ class CallLog:
|
|
|
136
289
|
redaction: CallRedaction | None = None,
|
|
137
290
|
max_values: int = DEFAULT_CALL_LOG_VALUES,
|
|
138
291
|
max_chars: int = DEFAULT_CALL_LOG_CHARS,
|
|
292
|
+
journal: CallJournal | None = None,
|
|
293
|
+
outbox: RequestOutbox | None = None,
|
|
139
294
|
):
|
|
140
295
|
if maximum is None:
|
|
141
296
|
raw = os.environ.get("WORLDS_CALL_LOG_MAX")
|
|
@@ -149,11 +304,22 @@ class CallLog:
|
|
|
149
304
|
self.values = 0
|
|
150
305
|
self.chars = 0
|
|
151
306
|
self.total = 0
|
|
307
|
+
self.journal = journal
|
|
308
|
+
self.outbox = outbox
|
|
309
|
+
self.webhooks = WebhookFeed()
|
|
152
310
|
self._lock = threading.Lock()
|
|
153
311
|
|
|
154
312
|
@classmethod
|
|
155
|
-
def for_spec(
|
|
156
|
-
|
|
313
|
+
def for_spec(
|
|
314
|
+
cls,
|
|
315
|
+
spec: Spec | None,
|
|
316
|
+
journal: CallJournal | None = None,
|
|
317
|
+
outbox: RequestOutbox | None = None,
|
|
318
|
+
secret: str | None = None,
|
|
319
|
+
) -> CallLog:
|
|
320
|
+
"""The call log of one session of the world ``spec`` declares (None: a world of
|
|
321
|
+
tools alone), whose strings never keep ``secret``, the session's credential."""
|
|
322
|
+
return cls(redaction=CallRedaction.of(spec, secret), journal=journal, outbox=outbox)
|
|
157
323
|
|
|
158
324
|
def record(
|
|
159
325
|
self,
|
|
@@ -169,16 +335,47 @@ class CallLog:
|
|
|
169
335
|
via: Via | None,
|
|
170
336
|
*,
|
|
171
337
|
shape: AnswerShape,
|
|
338
|
+
served: Served | None = None,
|
|
172
339
|
) -> None:
|
|
173
|
-
if self.maximum == 0:
|
|
174
|
-
return
|
|
175
340
|
door = {} if via is None else {"via": str(via)}
|
|
176
341
|
heading = {"operation": operation.name, "method": method}
|
|
177
342
|
try:
|
|
178
343
|
request, budget = self._route_request(operation, path, args, query_all, body)
|
|
179
344
|
except (RecursionError, RequestTooLarge):
|
|
180
345
|
request, budget = self._route_line(operation, path, args, query_all)
|
|
181
|
-
|
|
346
|
+
answered = _answered(status, shape.rows(answer), elapsed_s)
|
|
347
|
+
secret = served.secret if served else None
|
|
348
|
+
line = {
|
|
349
|
+
**heading,
|
|
350
|
+
"path": CallRedaction.path_without_secret(request["path"], secret),
|
|
351
|
+
"query": CallRedaction.without_secret(request["query"], secret),
|
|
352
|
+
"via": door.get("via"),
|
|
353
|
+
}
|
|
354
|
+
exchange = partial(Exchange.of, served, body, status, answer, self.redaction) if served else None
|
|
355
|
+
self._send(line, answered, exchange)
|
|
356
|
+
if self.maximum > 0:
|
|
357
|
+
self._append({**heading, **request, **door}, answered, budget)
|
|
358
|
+
|
|
359
|
+
def heard(
|
|
360
|
+
self, method: str, path: str, query: str, status: int, answer: Any, via: Via | None, served: Served
|
|
361
|
+
) -> None:
|
|
362
|
+
"""A request no route took (refused, or answered before routing): the ledger keeps it,
|
|
363
|
+
the grader's log does not."""
|
|
364
|
+
try:
|
|
365
|
+
kept_query = self.redaction.query({k: list(v) for k, v in urllib.parse.parse_qs(query).items()})
|
|
366
|
+
except (RecursionError, RequestTooLarge):
|
|
367
|
+
kept_query = {}
|
|
368
|
+
kept_query = CallRedaction.without_secret(kept_query, served.secret)
|
|
369
|
+
line = {"operation": None, "method": method, "path": self.redaction.raw_path(path, served.secret)}
|
|
370
|
+
exchange = partial(Exchange.of, served, None, status, answer, self.redaction)
|
|
371
|
+
door = None if via is None else str(via)
|
|
372
|
+
self._send({**line, "query": kept_query, "via": door}, _answered(status, None, 0.0), exchange)
|
|
373
|
+
|
|
374
|
+
def _send(self, line: dict, answered: dict, exchange: Callable[[], Exchange] | None) -> None:
|
|
375
|
+
if self.outbox is None or exchange is None:
|
|
376
|
+
return
|
|
377
|
+
found = {"rows": answered["rows"], "found": answered["found"], "durationMs": answered["elapsed_ms"]}
|
|
378
|
+
self.outbox.append({**line, **_bounded(line), **found}, exchange())
|
|
182
379
|
|
|
183
380
|
def _route_request(
|
|
184
381
|
self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
|
|
@@ -210,34 +407,93 @@ class CallLog:
|
|
|
210
407
|
"query": redaction.query(query_all, budget),
|
|
211
408
|
}
|
|
212
409
|
|
|
410
|
+
def refused_tool(self, tool: str, args: Any, refusal: dict) -> None:
|
|
411
|
+
"""A tool call the door refused before it ran (an unknown or disallowed name): the
|
|
412
|
+
ledger keeps it with the refusal it was answered, the grader's log does not."""
|
|
413
|
+
answer = {"error": refusal.get("code"), "message": refusal.get("message")}
|
|
414
|
+
exchange = partial(Exchange.of_tool, args, 400, answer, self.redaction)
|
|
415
|
+
line = {"operation": tool, "method": None, "path": None, "query": {}, "via": str(Via.TOOL)}
|
|
416
|
+
self._send(line, _answered(400, None, 0.0), exchange)
|
|
417
|
+
|
|
213
418
|
def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
|
|
419
|
+
heading = {"operation": tool, "method": None, "path": None}
|
|
420
|
+
answered = _answered(status, TOOL_ANSWER.rows(answer), elapsed_s)
|
|
421
|
+
exchange = partial(Exchange.of_tool, args, status, answer, self.redaction)
|
|
422
|
+
self._send({**heading, "query": {}, "via": str(Via.TOOL)}, answered, exchange)
|
|
214
423
|
if self.maximum == 0:
|
|
215
424
|
return
|
|
216
|
-
heading = {"operation": tool, "method": None, "path": None}
|
|
217
425
|
try:
|
|
218
426
|
request, budget = self._tool_request(args)
|
|
219
427
|
except (RecursionError, RequestTooLarge):
|
|
220
428
|
request, budget = dict(WITHHELD_TOOL), RequestBudget()
|
|
221
|
-
self._append({**heading, **request, "via": str(Via.TOOL)},
|
|
429
|
+
self._append({**heading, **request, "via": str(Via.TOOL)}, answered, budget)
|
|
430
|
+
|
|
431
|
+
def record_webhooks(self, deliveries: Iterable[Delivery]) -> None:
|
|
432
|
+
"""Each delivery the world sent, kept by the request ledger with ``via`` webhook and no
|
|
433
|
+
answer yet (status 0); the grader's call log keeps only what the agent called."""
|
|
434
|
+
sent = list(deliveries)
|
|
435
|
+
self.webhooks.append(sent)
|
|
436
|
+
for delivery in sent:
|
|
437
|
+
self._send_webhook(delivery, WebhookAnswer(delivery.id, 0, None, None), {"delivery": delivery.id}, "")
|
|
438
|
+
|
|
439
|
+
def answer_webhooks(self, answers: Iterable[WebhookAnswer]) -> int:
|
|
440
|
+
"""Each receiver's answer to a forwarded delivery, kept by the request ledger as one more
|
|
441
|
+
``via`` webhook entry under ``<event type>.answer`` (so a check counting the event never counts
|
|
442
|
+
it) with the receiver's status and ``query.answer`` naming the delivery; the
|
|
443
|
+
first answer to each delivery only, and none for an id the feed no longer holds."""
|
|
444
|
+
kept = 0
|
|
445
|
+
for answer in answers:
|
|
446
|
+
delivery = self.webhooks.answer(answer.delivery_id)
|
|
447
|
+
if delivery is None:
|
|
448
|
+
continue
|
|
449
|
+
self._send_webhook(delivery, answer, {"answer": delivery.id}, ANSWER_SUFFIX)
|
|
450
|
+
kept += 1
|
|
451
|
+
return kept
|
|
452
|
+
|
|
453
|
+
def webhook_page(self, outbox: WebhookOutbox | None, poll: WebhookPoll) -> dict:
|
|
454
|
+
"""A forwarder's poll: its answers kept first, then the subscriptions ``outbox`` reads (none
|
|
455
|
+
declared: ``declared`` false; at most :data:`SUBSCRIPTIONS_SHOWN`, each with how many deliveries
|
|
456
|
+
this feed sent it) and the page of deliveries."""
|
|
457
|
+
declared = outbox is not None and outbox.webhooks is not None
|
|
458
|
+
kept = self.answer_webhooks(poll.answers)
|
|
459
|
+
found = outbox.subscriptions() if outbox is not None and poll.subscriptions else []
|
|
460
|
+
shown = [{**sub, "sent": self.webhooks.sent_to(sub["key"])} for sub in found[:SUBSCRIPTIONS_SHOWN]]
|
|
461
|
+
page = self.webhooks.after(poll.after, poll.limit, poll.epoch, poll.tail, poll.targets)
|
|
462
|
+
listed = {"subscriptions": shown, "truncated": len(found) > len(shown)}
|
|
463
|
+
return {"declared": declared, **listed, "answered": kept, **page}
|
|
464
|
+
|
|
465
|
+
def _send_webhook(self, delivery: Delivery, answer: WebhookAnswer, query: dict, suffix: str) -> None:
|
|
466
|
+
path = self.redaction.raw_path(delivery.endpoint, self.redaction.secret)
|
|
467
|
+
line = {"operation": f"{delivery.event.type}{suffix}", "method": "POST", "path": path, "query": query}
|
|
468
|
+
exchange = partial(Exchange.of_webhook, delivery, answer, self.redaction)
|
|
469
|
+
answered = _answered(answer.status, None, (answer.duration_ms or 0.0) / 1000)
|
|
470
|
+
self._send({**line, "via": str(Via.WEBHOOK)}, answered, exchange)
|
|
222
471
|
|
|
223
472
|
def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
|
|
224
473
|
budget = RequestBudget()
|
|
225
474
|
logged_args = self.redaction.value(args, budget)
|
|
226
475
|
return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
|
|
227
476
|
|
|
228
|
-
def _append(self, request: dict,
|
|
229
|
-
rows = None if status >= 400 else rows
|
|
230
|
-
answered = {
|
|
231
|
-
"status": status,
|
|
232
|
-
"rows": rows,
|
|
233
|
-
"found": _found(status, rows),
|
|
234
|
-
"elapsed_ms": round(elapsed_s * 1000, 1),
|
|
235
|
-
}
|
|
477
|
+
def _append(self, request: dict, answered: dict, budget: RequestBudget) -> None:
|
|
236
478
|
at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
237
479
|
cost = budget.spent()
|
|
238
480
|
with self._lock:
|
|
239
481
|
self.total += 1
|
|
240
|
-
|
|
482
|
+
entry = {"seq": self.total, "at": at, **request, **answered}
|
|
483
|
+
self._keep(entry, cost)
|
|
484
|
+
self._journal(entry)
|
|
485
|
+
|
|
486
|
+
def _journal(self, entry: dict) -> None:
|
|
487
|
+
if self.journal is None:
|
|
488
|
+
return
|
|
489
|
+
line = CallJournal.line(entry)
|
|
490
|
+
if self.journal.appended + len(line) > 2 * self.max_chars:
|
|
491
|
+
return self._rewrite_journal()
|
|
492
|
+
self.journal.append(line)
|
|
493
|
+
|
|
494
|
+
def _rewrite_journal(self) -> None:
|
|
495
|
+
if self.journal is not None:
|
|
496
|
+
self.journal.rewrite(self.entries, 4 * self.max_chars)
|
|
241
497
|
|
|
242
498
|
def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
|
|
243
499
|
self.entries.append(entry)
|
|
@@ -281,6 +537,7 @@ class CallLog:
|
|
|
281
537
|
for call, cost in zip(calls, costs):
|
|
282
538
|
self._keep(call, cost)
|
|
283
539
|
self.total = max([self.total, *(call["seq"] for call in calls)])
|
|
540
|
+
self._rewrite_journal()
|
|
284
541
|
|
|
285
542
|
def clear(self) -> None:
|
|
286
543
|
with self._lock:
|
|
@@ -289,3 +546,4 @@ class CallLog:
|
|
|
289
546
|
self.values = 0
|
|
290
547
|
self.chars = 0
|
|
291
548
|
self.total = 0
|
|
549
|
+
self._rewrite_journal()
|