@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
"""The call log: every call one session answered, by every door, in order."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import collections
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import stat
|
|
9
|
+
import threading
|
|
10
|
+
import urllib.parse
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from enum import StrEnum
|
|
14
|
+
from functools import partial
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Callable, Iterable
|
|
17
|
+
|
|
18
|
+
from .call_capture import Exchange, Served
|
|
19
|
+
from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
|
|
20
|
+
from .connector import Operation, Spec, _dig
|
|
21
|
+
from .handler_errors import log_line
|
|
22
|
+
from .tasks import WITHHELD_FIELD
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from .request_ledger import RequestOutbox
|
|
26
|
+
|
|
27
|
+
METHOD_CHARS = 32
|
|
28
|
+
OPERATION_CHARS = 512
|
|
29
|
+
PATH_CHARS = 64 << 10
|
|
30
|
+
LINE_BOUNDS = {"method": METHOD_CHARS, "operation": OPERATION_CHARS, "path": PATH_CHARS}
|
|
31
|
+
DEFAULT_CALL_LOG_MAX = 5000
|
|
32
|
+
DEFAULT_CALL_LOG_VALUES = 200_000
|
|
33
|
+
DEFAULT_CALL_LOG_CHARS = 8 << 20
|
|
34
|
+
WITHHELD_ROUTE = {
|
|
35
|
+
"args": REDACTED,
|
|
36
|
+
"query": {},
|
|
37
|
+
"body": None,
|
|
38
|
+
"selectors": [],
|
|
39
|
+
WITHHELD_FIELD: ["path", "args", "query", "body", "selectors"],
|
|
40
|
+
}
|
|
41
|
+
WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def read_tail(path: str | Path, max_bytes: int | None, dir_fd: int | None = None) -> bytes:
|
|
45
|
+
"""The newest ``max_bytes`` of the file at ``path`` (all of it when None; relative to ``dir_fd`` when
|
|
46
|
+
given) as tenant code could have left it: no link followed, no FIFO waited on, anything but a regular
|
|
47
|
+
file read as empty. Raises OSError when it cannot be opened or read; its descriptor is closed on
|
|
48
|
+
every path."""
|
|
49
|
+
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dir_fd)
|
|
50
|
+
try:
|
|
51
|
+
found = os.fstat(descriptor)
|
|
52
|
+
if not stat.S_ISREG(found.st_mode):
|
|
53
|
+
return b""
|
|
54
|
+
size = found.st_size if max_bytes is None else min(max_bytes, found.st_size)
|
|
55
|
+
os.lseek(descriptor, found.st_size - size, os.SEEK_SET)
|
|
56
|
+
return _read_up_to(descriptor, size)
|
|
57
|
+
finally:
|
|
58
|
+
os.close(descriptor)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _read_up_to(descriptor: int, max_bytes: int) -> bytes:
|
|
62
|
+
chunks: list[bytes] = []
|
|
63
|
+
left = max_bytes
|
|
64
|
+
while left > 0:
|
|
65
|
+
chunk = os.read(descriptor, min(left, 1 << 20))
|
|
66
|
+
if not chunk:
|
|
67
|
+
break
|
|
68
|
+
chunks.append(chunk)
|
|
69
|
+
left -= len(chunk)
|
|
70
|
+
return b"".join(chunks)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _children(node: Any) -> Any:
|
|
74
|
+
if isinstance(node, dict):
|
|
75
|
+
return node.values()
|
|
76
|
+
if isinstance(node, (list, tuple)):
|
|
77
|
+
return node
|
|
78
|
+
return ()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _strings_in(node: Any) -> list[str]:
|
|
82
|
+
"""Every string in a JSON value, however nested, once each, in document order."""
|
|
83
|
+
found: dict[str, None] = {}
|
|
84
|
+
_collect_strings(node, found)
|
|
85
|
+
return list(found)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _collect_strings(node: Any, found: dict[str, None]) -> None:
|
|
89
|
+
if isinstance(node, str) and node != REDACTED:
|
|
90
|
+
found[node] = None
|
|
91
|
+
for value in _children(node):
|
|
92
|
+
_collect_strings(value, found)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _counted(found: Any) -> int | None:
|
|
96
|
+
if isinstance(found, list):
|
|
97
|
+
return len(found)
|
|
98
|
+
if isinstance(found, dict):
|
|
99
|
+
return 1 if found else 0
|
|
100
|
+
return 0 if found is None else None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _found(status: int, rows: int | None) -> bool | None:
|
|
104
|
+
if status >= 400:
|
|
105
|
+
return False
|
|
106
|
+
return None if rows is None else rows > 0
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _bounded(line: dict) -> dict:
|
|
110
|
+
"""A ledger line's free-text fields cut to the ledger's bounds (``limits`` in
|
|
111
|
+
conformance/world-request-log.json), so the platform keeps the request."""
|
|
112
|
+
return {
|
|
113
|
+
field: value[:size] if isinstance(value, str) else value
|
|
114
|
+
for field, size in LINE_BOUNDS.items()
|
|
115
|
+
for value in [line.get(field)]
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _answered(status: int, rows: int | None, elapsed_s: float) -> dict:
|
|
120
|
+
rows = None if status >= 400 else rows
|
|
121
|
+
return {"status": status, "rows": rows, "found": _found(status, rows), "elapsed_ms": round(elapsed_s * 1000, 1)}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class Via(StrEnum):
|
|
125
|
+
"""The door a call came in by, set by the server from where it arrived and
|
|
126
|
+
never from a request header. It names the door, not the actor: a token
|
|
127
|
+
holder calling the ui origin's prefix without a browser is ``ui``."""
|
|
128
|
+
|
|
129
|
+
API = "api"
|
|
130
|
+
UI = "ui"
|
|
131
|
+
TOOL = "tool"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass(frozen=True)
|
|
135
|
+
class AnswerShape:
|
|
136
|
+
"""Where an answer carries its rows, as the route declares it: a list is
|
|
137
|
+
its rows; under the envelope key the route serves, a list; at the
|
|
138
|
+
operation's results path, a list, one object (one row) or nothing (none);
|
|
139
|
+
a ``single`` route's object is one row. Nothing answered is no rows. Any
|
|
140
|
+
other answer (a tool's object, a handler's own shape) carries rows nobody
|
|
141
|
+
declared: unknown, ``None``."""
|
|
142
|
+
|
|
143
|
+
envelope: str = ""
|
|
144
|
+
results: str = ""
|
|
145
|
+
single: bool = False
|
|
146
|
+
|
|
147
|
+
def rows(self, answer: Any) -> int | None:
|
|
148
|
+
if isinstance(answer, list):
|
|
149
|
+
return len(answer)
|
|
150
|
+
if answer is None:
|
|
151
|
+
return 0
|
|
152
|
+
if not isinstance(answer, dict):
|
|
153
|
+
return None
|
|
154
|
+
listed = _dig(answer, self.envelope) if self.envelope else None
|
|
155
|
+
if isinstance(listed, list):
|
|
156
|
+
return len(listed)
|
|
157
|
+
if self.results:
|
|
158
|
+
return _counted(_dig(answer, self.results))
|
|
159
|
+
if self.single:
|
|
160
|
+
return 1 if answer else 0
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
TOOL_ANSWER = AnswerShape()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class CallJournal:
|
|
168
|
+
"""A version worker's session's call log on disk beside its store, one JSON line
|
|
169
|
+
per entry as it is kept, so a worker that dies leaves its calls with its rows and
|
|
170
|
+
the next worker resumes both. A call is journaled after its transaction commits:
|
|
171
|
+
a worker that dies in between keeps the call's rows and loses its entry. Once what
|
|
172
|
+
was appended since the last rewrite passes the log's own budget, the file is
|
|
173
|
+
rewritten from the newest of the log's entries that fit ``budget`` bytes: bounded
|
|
174
|
+
on disk whatever the calls carry, and the whole log unless its text is mostly
|
|
175
|
+
escaped in JSON (a control character takes six bytes)."""
|
|
176
|
+
|
|
177
|
+
def __init__(self, path: Path) -> None:
|
|
178
|
+
self.path = Path(path)
|
|
179
|
+
self.appended = 0
|
|
180
|
+
|
|
181
|
+
@staticmethod
|
|
182
|
+
def line(entry: dict) -> bytes:
|
|
183
|
+
return (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
|
|
184
|
+
|
|
185
|
+
def append(self, line: bytes) -> None:
|
|
186
|
+
try:
|
|
187
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
188
|
+
with open(self.path, "ab") as journal:
|
|
189
|
+
journal.write(line)
|
|
190
|
+
except OSError as error:
|
|
191
|
+
return self._unwritten(error)
|
|
192
|
+
self.appended += len(line)
|
|
193
|
+
|
|
194
|
+
def rewrite(self, entries: Iterable[dict], budget: int) -> None:
|
|
195
|
+
staging = self.path.with_name(f".{self.path.name}.next")
|
|
196
|
+
try:
|
|
197
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
staging.write_bytes(b"".join(self._newest(entries, budget)))
|
|
199
|
+
os.replace(staging, self.path)
|
|
200
|
+
except OSError as error:
|
|
201
|
+
return self._unwritten(error)
|
|
202
|
+
self.appended = 0
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def _newest(cls, entries: Iterable[dict], budget: int) -> list[bytes]:
|
|
206
|
+
kept: list[bytes] = []
|
|
207
|
+
for line in map(cls.line, reversed(list(entries))):
|
|
208
|
+
budget -= len(line)
|
|
209
|
+
if budget < 0:
|
|
210
|
+
break
|
|
211
|
+
kept.append(line)
|
|
212
|
+
return kept[::-1]
|
|
213
|
+
|
|
214
|
+
def _unwritten(self, error: OSError) -> None:
|
|
215
|
+
log_line(f"call journal {self.path.name} not written: {error}")
|
|
216
|
+
|
|
217
|
+
@staticmethod
|
|
218
|
+
def read(path: Path) -> list[dict]:
|
|
219
|
+
"""The entries a journal holds, oldest first, read by the worker that continues the session
|
|
220
|
+
from it: :meth:`read_left` over the whole file, so a worker killed mid-write or a tenant's
|
|
221
|
+
directory, link or FIFO at the path costs that journal's entries, never the reopen."""
|
|
222
|
+
return CallJournal._read(path, None)
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def read_left(path: Path, max_bytes: int) -> list[dict]:
|
|
226
|
+
"""The newest ``max_bytes`` of a journal a dead worker left, read by the supervisor: the
|
|
227
|
+
worker ran tenant code, so a link, a FIFO, a directory or anything but a regular file
|
|
228
|
+
reads as empty, and so does a file that cannot be read."""
|
|
229
|
+
return CallJournal._read(path, max_bytes)
|
|
230
|
+
|
|
231
|
+
@staticmethod
|
|
232
|
+
def _read(path: Path, max_bytes: int | None) -> list[dict]:
|
|
233
|
+
try:
|
|
234
|
+
text = read_tail(path, max_bytes).decode("utf-8", "replace")
|
|
235
|
+
except OSError:
|
|
236
|
+
return []
|
|
237
|
+
return CallJournal._entries(text)
|
|
238
|
+
|
|
239
|
+
@staticmethod
|
|
240
|
+
def _entries(text: str) -> list[dict]:
|
|
241
|
+
"""The entries of a journal's text, one to a line. Lines end at ``\\n`` alone: ``str.splitlines``
|
|
242
|
+
also cuts at U+2028, U+2029 and U+0085, which :meth:`line` leaves raw inside an entry."""
|
|
243
|
+
return [entry for entry in map(CallJournal._entry, text.split("\n")) if entry is not None]
|
|
244
|
+
|
|
245
|
+
@staticmethod
|
|
246
|
+
def _entry(line: str) -> dict | None:
|
|
247
|
+
try:
|
|
248
|
+
entry = json.loads(line)
|
|
249
|
+
except (ValueError, RecursionError):
|
|
250
|
+
return None
|
|
251
|
+
return entry if isinstance(entry, dict) and isinstance(entry.get("seq"), int) else None
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class CallLog:
|
|
255
|
+
"""The calls one session answered, in order: what the agent asked (a
|
|
256
|
+
route's operation, method, path, path args, query and body; a tool's name
|
|
257
|
+
and arguments), the door it came in by (``via``) and what came back
|
|
258
|
+
(status; ``rows``, the rows the answer carried as :class:`AnswerShape`
|
|
259
|
+
reads them; elapsed). ``selectors`` is every string the request carried
|
|
260
|
+
wherever it sat, and ``found`` is "answered with rows", so a grader's
|
|
261
|
+
scalar ``where`` can ask either without the vendor's request shape.
|
|
262
|
+
:class:`CallRedaction` decides what of a request is kept. A request too
|
|
263
|
+
large or too deep to keep names what it lacks in ``withheld``: a route call
|
|
264
|
+
keeps its request line (path, path arguments, query), which HTTP bounds,
|
|
265
|
+
and withholds its body (``["body", "selectors"]``); a tool call withholds its
|
|
266
|
+
arguments. Bounded at ``maximum`` entries (``WORLDS_CALL_LOG_MAX``, default
|
|
267
|
+
:data:`DEFAULT_CALL_LOG_MAX`; ``0`` keeps none) and at what its entries hold,
|
|
268
|
+
``max_values`` values and ``max_chars`` characters (the sum of their
|
|
269
|
+
:class:`RequestBudget` spends; about 60 MB of memory at most), the oldest
|
|
270
|
+
dropped first; ``total`` counts past the bound, so the first kept ``seq``
|
|
271
|
+
says how many are gone. Every session's ``state`` and grader state carry it
|
|
272
|
+
(:meth:`into_state`), ``serve()`` answers it at ``/__world/calls`` to its admin
|
|
273
|
+
token, a checkpoint carries it (:meth:`resume`), and a task's
|
|
274
|
+
``verify.py`` is handed it
|
|
275
|
+
as ``calls`` beside the rows, because a read-only investigation leaves no
|
|
276
|
+
row behind and its calls are the only evidence of what it did.
|
|
277
|
+
|
|
278
|
+
``outbox`` is the session's half of the world request ledger: every call recorded
|
|
279
|
+
here, and every request no route took (:meth:`heard`), goes there with its
|
|
280
|
+
exchange (:class:`Exchange`) whatever this log's own bounds keep."""
|
|
281
|
+
|
|
282
|
+
def __init__(
|
|
283
|
+
self,
|
|
284
|
+
maximum: int | None = None,
|
|
285
|
+
redaction: CallRedaction | None = None,
|
|
286
|
+
max_values: int = DEFAULT_CALL_LOG_VALUES,
|
|
287
|
+
max_chars: int = DEFAULT_CALL_LOG_CHARS,
|
|
288
|
+
journal: CallJournal | None = None,
|
|
289
|
+
outbox: RequestOutbox | None = None,
|
|
290
|
+
):
|
|
291
|
+
if maximum is None:
|
|
292
|
+
raw = os.environ.get("WORLDS_CALL_LOG_MAX")
|
|
293
|
+
maximum = int(raw) if raw else DEFAULT_CALL_LOG_MAX
|
|
294
|
+
self.maximum = max(0, int(maximum))
|
|
295
|
+
self.max_values = max_values
|
|
296
|
+
self.max_chars = max_chars
|
|
297
|
+
self.redaction = redaction or CallRedaction()
|
|
298
|
+
self.entries: collections.deque[dict] = collections.deque()
|
|
299
|
+
self._costs: collections.deque[tuple[int, int]] = collections.deque()
|
|
300
|
+
self.values = 0
|
|
301
|
+
self.chars = 0
|
|
302
|
+
self.total = 0
|
|
303
|
+
self.journal = journal
|
|
304
|
+
self.outbox = outbox
|
|
305
|
+
self._lock = threading.Lock()
|
|
306
|
+
|
|
307
|
+
@classmethod
|
|
308
|
+
def for_spec(
|
|
309
|
+
cls,
|
|
310
|
+
spec: Spec | None,
|
|
311
|
+
journal: CallJournal | None = None,
|
|
312
|
+
outbox: RequestOutbox | None = None,
|
|
313
|
+
secret: str | None = None,
|
|
314
|
+
) -> CallLog:
|
|
315
|
+
"""The call log of one session of the world ``spec`` declares (None: a world of
|
|
316
|
+
tools alone), whose strings never keep ``secret``, the session's credential."""
|
|
317
|
+
return cls(redaction=CallRedaction.of(spec, secret), journal=journal, outbox=outbox)
|
|
318
|
+
|
|
319
|
+
def record(
|
|
320
|
+
self,
|
|
321
|
+
operation: Operation,
|
|
322
|
+
method: str,
|
|
323
|
+
path: str,
|
|
324
|
+
args: dict,
|
|
325
|
+
query_all: dict,
|
|
326
|
+
body: Any,
|
|
327
|
+
status: int,
|
|
328
|
+
answer: Any,
|
|
329
|
+
elapsed_s: float,
|
|
330
|
+
via: Via | None,
|
|
331
|
+
*,
|
|
332
|
+
shape: AnswerShape,
|
|
333
|
+
served: Served | None = None,
|
|
334
|
+
) -> None:
|
|
335
|
+
door = {} if via is None else {"via": str(via)}
|
|
336
|
+
heading = {"operation": operation.name, "method": method}
|
|
337
|
+
try:
|
|
338
|
+
request, budget = self._route_request(operation, path, args, query_all, body)
|
|
339
|
+
except (RecursionError, RequestTooLarge):
|
|
340
|
+
request, budget = self._route_line(operation, path, args, query_all)
|
|
341
|
+
answered = _answered(status, shape.rows(answer), elapsed_s)
|
|
342
|
+
secret = served.secret if served else None
|
|
343
|
+
line = {
|
|
344
|
+
**heading,
|
|
345
|
+
"path": CallRedaction.path_without_secret(request["path"], secret),
|
|
346
|
+
"query": CallRedaction.without_secret(request["query"], secret),
|
|
347
|
+
"via": door.get("via"),
|
|
348
|
+
}
|
|
349
|
+
exchange = partial(Exchange.of, served, body, status, answer, self.redaction) if served else None
|
|
350
|
+
self._send(line, answered, exchange)
|
|
351
|
+
if self.maximum > 0:
|
|
352
|
+
self._append({**heading, **request, **door}, answered, budget)
|
|
353
|
+
|
|
354
|
+
def heard(
|
|
355
|
+
self, method: str, path: str, query: str, status: int, answer: Any, via: Via | None, served: Served
|
|
356
|
+
) -> None:
|
|
357
|
+
"""A request no route took (refused, or answered before routing): the ledger keeps it,
|
|
358
|
+
the grader's log does not."""
|
|
359
|
+
try:
|
|
360
|
+
kept_query = self.redaction.query({k: list(v) for k, v in urllib.parse.parse_qs(query).items()})
|
|
361
|
+
except (RecursionError, RequestTooLarge):
|
|
362
|
+
kept_query = {}
|
|
363
|
+
kept_query = CallRedaction.without_secret(kept_query, served.secret)
|
|
364
|
+
line = {"operation": None, "method": method, "path": self.redaction.raw_path(path, served.secret)}
|
|
365
|
+
exchange = partial(Exchange.of, served, None, status, answer, self.redaction)
|
|
366
|
+
door = None if via is None else str(via)
|
|
367
|
+
self._send({**line, "query": kept_query, "via": door}, _answered(status, None, 0.0), exchange)
|
|
368
|
+
|
|
369
|
+
def _send(self, line: dict, answered: dict, exchange: Callable[[], Exchange] | None) -> None:
|
|
370
|
+
if self.outbox is None or exchange is None:
|
|
371
|
+
return
|
|
372
|
+
found = {"rows": answered["rows"], "found": answered["found"], "durationMs": answered["elapsed_ms"]}
|
|
373
|
+
self.outbox.append({**line, **_bounded(line), **found}, exchange())
|
|
374
|
+
|
|
375
|
+
def _route_request(
|
|
376
|
+
self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
|
|
377
|
+
) -> tuple[dict, RequestBudget]:
|
|
378
|
+
budget = RequestBudget()
|
|
379
|
+
line = self._request_line(operation, path, args, query_all, budget)
|
|
380
|
+
logged_body = self.redaction.value(body, budget) if isinstance(body, (dict, list)) and body else None
|
|
381
|
+
selectors = _strings_in([line["args"], line["query"], logged_body])
|
|
382
|
+
return {**line, "body": logged_body, "selectors": selectors}, budget
|
|
383
|
+
|
|
384
|
+
def _route_line(self, operation: Operation, path: str, args: dict, query_all: dict) -> tuple[dict, RequestBudget]:
|
|
385
|
+
budget = RequestBudget()
|
|
386
|
+
try:
|
|
387
|
+
line = self._request_line(operation, path, args, query_all, budget)
|
|
388
|
+
except (RecursionError, RequestTooLarge):
|
|
389
|
+
return {"path": operation.path, **WITHHELD_ROUTE}, RequestBudget()
|
|
390
|
+
selectors = _strings_in([line["args"], line["query"]])
|
|
391
|
+
return {**line, "body": None, "selectors": selectors, WITHHELD_FIELD: ["body", "selectors"]}, budget
|
|
392
|
+
|
|
393
|
+
def _request_line(
|
|
394
|
+
self, operation: Operation, path: str, args: dict, query_all: dict, budget: RequestBudget
|
|
395
|
+
) -> dict:
|
|
396
|
+
redaction = self.redaction
|
|
397
|
+
logged_path = redaction.path(operation, path, args)
|
|
398
|
+
budget.spend(1, len(logged_path))
|
|
399
|
+
return {
|
|
400
|
+
"path": logged_path,
|
|
401
|
+
"args": redaction.value(dict(args), budget),
|
|
402
|
+
"query": redaction.query(query_all, budget),
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
def refused_tool(self, tool: str, args: Any, refusal: dict) -> None:
|
|
406
|
+
"""A tool call the door refused before it ran (an unknown or disallowed name): the
|
|
407
|
+
ledger keeps it with the refusal it was answered, the grader's log does not."""
|
|
408
|
+
answer = {"error": refusal.get("code"), "message": refusal.get("message")}
|
|
409
|
+
exchange = partial(Exchange.of_tool, args, 400, answer, self.redaction)
|
|
410
|
+
line = {"operation": tool, "method": None, "path": None, "query": {}, "via": str(Via.TOOL)}
|
|
411
|
+
self._send(line, _answered(400, None, 0.0), exchange)
|
|
412
|
+
|
|
413
|
+
def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
|
|
414
|
+
heading = {"operation": tool, "method": None, "path": None}
|
|
415
|
+
answered = _answered(status, TOOL_ANSWER.rows(answer), elapsed_s)
|
|
416
|
+
exchange = partial(Exchange.of_tool, args, status, answer, self.redaction)
|
|
417
|
+
self._send({**heading, "query": {}, "via": str(Via.TOOL)}, answered, exchange)
|
|
418
|
+
if self.maximum == 0:
|
|
419
|
+
return
|
|
420
|
+
try:
|
|
421
|
+
request, budget = self._tool_request(args)
|
|
422
|
+
except (RecursionError, RequestTooLarge):
|
|
423
|
+
request, budget = dict(WITHHELD_TOOL), RequestBudget()
|
|
424
|
+
self._append({**heading, **request, "via": str(Via.TOOL)}, answered, budget)
|
|
425
|
+
|
|
426
|
+
def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
|
|
427
|
+
budget = RequestBudget()
|
|
428
|
+
logged_args = self.redaction.value(args, budget)
|
|
429
|
+
return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
|
|
430
|
+
|
|
431
|
+
def _append(self, request: dict, answered: dict, budget: RequestBudget) -> None:
|
|
432
|
+
at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
433
|
+
cost = budget.spent()
|
|
434
|
+
with self._lock:
|
|
435
|
+
self.total += 1
|
|
436
|
+
entry = {"seq": self.total, "at": at, **request, **answered}
|
|
437
|
+
self._keep(entry, cost)
|
|
438
|
+
self._journal(entry)
|
|
439
|
+
|
|
440
|
+
def _journal(self, entry: dict) -> None:
|
|
441
|
+
if self.journal is None:
|
|
442
|
+
return
|
|
443
|
+
line = CallJournal.line(entry)
|
|
444
|
+
if self.journal.appended + len(line) > 2 * self.max_chars:
|
|
445
|
+
return self._rewrite_journal()
|
|
446
|
+
self.journal.append(line)
|
|
447
|
+
|
|
448
|
+
def _rewrite_journal(self) -> None:
|
|
449
|
+
if self.journal is not None:
|
|
450
|
+
self.journal.rewrite(self.entries, 4 * self.max_chars)
|
|
451
|
+
|
|
452
|
+
def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
|
|
453
|
+
self.entries.append(entry)
|
|
454
|
+
self._costs.append(cost)
|
|
455
|
+
self.values += cost[0]
|
|
456
|
+
self.chars += cost[1]
|
|
457
|
+
while len(self.entries) > self.maximum or (len(self.entries) > 1 and self._over_budget()):
|
|
458
|
+
self.entries.popleft()
|
|
459
|
+
values, chars = self._costs.popleft()
|
|
460
|
+
self.values -= values
|
|
461
|
+
self.chars -= chars
|
|
462
|
+
|
|
463
|
+
def _over_budget(self) -> bool:
|
|
464
|
+
return self.values > self.max_values or self.chars > self.max_chars
|
|
465
|
+
|
|
466
|
+
def _held(self, call: dict) -> tuple[int, int]:
|
|
467
|
+
budget = RequestBudget()
|
|
468
|
+
try:
|
|
469
|
+
self.redaction.value([call.get(field) for field in ("path", "args", "query", "body")], budget)
|
|
470
|
+
except (RecursionError, RequestTooLarge):
|
|
471
|
+
return MAX_LOGGED_VALUES, MAX_LOGGED_CHARS
|
|
472
|
+
return budget.spent()
|
|
473
|
+
|
|
474
|
+
def describe(self) -> dict:
|
|
475
|
+
with self._lock:
|
|
476
|
+
return {"total": self.total, "kept": len(self.entries), "max": self.maximum, "calls": list(self.entries)}
|
|
477
|
+
|
|
478
|
+
def into_state(self, state: dict) -> dict:
|
|
479
|
+
"""``state`` with the calls beside its rows, as every session's ``state`` and grader state carry them."""
|
|
480
|
+
state["calls"] = self.describe()["calls"]
|
|
481
|
+
return state
|
|
482
|
+
|
|
483
|
+
def resume(self, calls: list[dict]) -> None:
|
|
484
|
+
"""Carry on from the calls a checkpoint kept: the same entries, numbered on
|
|
485
|
+
from the last of them, each charged what it holds. A log that keeps
|
|
486
|
+
nothing keeps none of them either."""
|
|
487
|
+
if self.maximum == 0:
|
|
488
|
+
return
|
|
489
|
+
costs = [self._held(call) for call in calls]
|
|
490
|
+
with self._lock:
|
|
491
|
+
for call, cost in zip(calls, costs):
|
|
492
|
+
self._keep(call, cost)
|
|
493
|
+
self.total = max([self.total, *(call["seq"] for call in calls)])
|
|
494
|
+
self._rewrite_journal()
|
|
495
|
+
|
|
496
|
+
def clear(self) -> None:
|
|
497
|
+
with self._lock:
|
|
498
|
+
self.entries.clear()
|
|
499
|
+
self._costs.clear()
|
|
500
|
+
self.values = 0
|
|
501
|
+
self.chars = 0
|
|
502
|
+
self.total = 0
|
|
503
|
+
self._rewrite_journal()
|