@withgateway/sdk 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1128 -783
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +1130 -785
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +47 -18
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +47 -18
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +179 -43
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +176 -43
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +47 -18
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +47 -18
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +184 -85
- package/dist/worlds/index.d.ts +184 -85
- package/dist/worlds/index.js +176 -43
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +176 -43
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_world.py +24 -1
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/api.py +24 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +13 -1
- package/runtime/gatewaysdk/environment/schema/call_log.py +46 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +6 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +32 -8
- package/runtime/gatewaysdk/environment/schema/connector.py +5 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +9 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +2 -1
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +27 -1
- package/runtime/gatewaysdk/environment/schema/journal.py +4 -3
- package/runtime/gatewaysdk/environment/schema/local.py +1 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +16 -219
- package/runtime/gatewaysdk/environment/schema/runtime.py +7 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -22
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +7 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +14 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +36 -5
- package/runtime/gatewaysdk/environment/schema/snapshot.py +2 -2
- package/runtime/gatewaysdk/environment/schema/tools_world.py +19 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +6 -0
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/world_host.py +20 -3
- package/runtime/gatewaysdk/environment/schema/world_tests.py +74 -0
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +19 -22
- package/skills/connector-schema-authoring/connector-worlds.md +153 -11
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/SKILL.md +7 -9
- package/skills/worlds-getting-started/SKILL.md +3 -3
- package/skills/worlds-getting-started/references/cli-mcp-map.md +14 -11
- package/skills/worlds-getting-started/references/gotchas.md +36 -5
package/package.json
CHANGED
|
@@ -18,6 +18,7 @@ import importlib.util
|
|
|
18
18
|
import inspect
|
|
19
19
|
import sys
|
|
20
20
|
import uuid
|
|
21
|
+
from contextlib import ExitStack
|
|
21
22
|
from dataclasses import dataclass, field
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
from typing import Any, Callable, Dict, List, Optional
|
|
@@ -124,8 +125,8 @@ def load_handlers(bundle_path: Path, manifest: EnvManifest) -> Dict[str, Callabl
|
|
|
124
125
|
handler_files = sorted(tools_dir.glob("*.py")) if tools_dir.is_dir() else []
|
|
125
126
|
load_token = runtime._load_registry.set(registry)
|
|
126
127
|
strict_token = runtime._strict_registration.set(contract is not None)
|
|
127
|
-
|
|
128
|
-
|
|
128
|
+
installed = ExitStack()
|
|
129
|
+
installed.enter_context(runtime.installed(runtime))
|
|
129
130
|
# Generated bundles use intra-bundle imports ("from tools._db import …"),
|
|
130
131
|
# so the bundle root must be importable for the load's dynamic extent.
|
|
131
132
|
bundle_root = str(bundle_path)
|
|
@@ -158,10 +159,7 @@ def load_handlers(bundle_path: Path, manifest: EnvManifest) -> Dict[str, Callabl
|
|
|
158
159
|
modules.append(module)
|
|
159
160
|
modules_by_stem[handler_file.stem] = module
|
|
160
161
|
finally:
|
|
161
|
-
|
|
162
|
-
sys.modules["gateway_env"] = saved
|
|
163
|
-
else:
|
|
164
|
-
sys.modules.pop("gateway_env", None)
|
|
162
|
+
installed.close()
|
|
165
163
|
if path_added:
|
|
166
164
|
try:
|
|
167
165
|
sys.path.remove(bundle_root)
|
|
@@ -20,6 +20,7 @@ if TYPE_CHECKING: # pragma: no cover
|
|
|
20
20
|
from .core import Environment
|
|
21
21
|
from .schema.journal import JournalUse, WorldJournal
|
|
22
22
|
from .schema.session_files import ProvenTemplate, SessionExport
|
|
23
|
+
from .schema.webhooks import WebhookOutbox, Webhooks
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
@dataclass
|
|
@@ -120,6 +121,7 @@ class World:
|
|
|
120
121
|
self.contract = template.contract if template is not None else self._bundle_contract()
|
|
121
122
|
source, audit = (Path(snapshot), True) if snapshot is not None else self._seed()
|
|
122
123
|
self._backing = WorldBacking.branch(source, self.db_path, self.contract, audit=audit)
|
|
124
|
+
self.webhooks = self._webhook_outbox(None)
|
|
123
125
|
|
|
124
126
|
@staticmethod
|
|
125
127
|
def database_file(directory: Path, world_id: str) -> Path:
|
|
@@ -133,6 +135,24 @@ class World:
|
|
|
133
135
|
|
|
134
136
|
return ProvenTemplate.required(template)
|
|
135
137
|
|
|
138
|
+
def serve_webhooks(self, webhooks: "Webhooks | None") -> None:
|
|
139
|
+
"""Deliveries this world's tools stage go to the subscriptions ``webhooks`` declares."""
|
|
140
|
+
if webhooks is not None and self.contract is not None:
|
|
141
|
+
webhooks.check_contract(self.contract.entities)
|
|
142
|
+
self.webhooks = self._webhook_outbox(webhooks if self.contract is not None else None)
|
|
143
|
+
|
|
144
|
+
def _webhook_outbox(self, webhooks: "Webhooks | None") -> "WebhookOutbox":
|
|
145
|
+
from .schema.api import ApiContext
|
|
146
|
+
from .schema.primary_key import PrimaryKey
|
|
147
|
+
from .schema.webhooks import WebhookOutbox
|
|
148
|
+
|
|
149
|
+
return WebhookOutbox(
|
|
150
|
+
webhooks,
|
|
151
|
+
lambda entity: list(self.store.list(entity)),
|
|
152
|
+
lambda entity, row: PrimaryKey.in_contract(self.contract, entity).key(row),
|
|
153
|
+
ApiContext.clock,
|
|
154
|
+
)
|
|
155
|
+
|
|
136
156
|
def _bundle_contract(self) -> Any:
|
|
137
157
|
from .schema import bundle_mode, check_bundle
|
|
138
158
|
|
|
@@ -156,7 +176,7 @@ class World:
|
|
|
156
176
|
raise ToolRouteError(f"tool '{tool_name}' has no declared env handler")
|
|
157
177
|
supplied = ({} if self.contract else None) if tool_input is _NO_INPUT else tool_input
|
|
158
178
|
started = time.perf_counter()
|
|
159
|
-
with self._backing.transaction():
|
|
179
|
+
with self.webhooks.transaction(), self._backing.transaction():
|
|
160
180
|
if self.contract is not None:
|
|
161
181
|
validate(
|
|
162
182
|
self.contract.tools[tool_name]["input_schema"],
|
|
@@ -165,6 +185,7 @@ class World:
|
|
|
165
185
|
)
|
|
166
186
|
token = runtime._active_connection.set(self._backing.handler_connection())
|
|
167
187
|
store_token = runtime._active_store.set(self.store)
|
|
188
|
+
webhooks_token = runtime._active_webhooks.set(self.webhooks)
|
|
168
189
|
try:
|
|
169
190
|
if isinstance(supplied, Mapping):
|
|
170
191
|
result = handler(**supplied)
|
|
@@ -181,6 +202,7 @@ class World:
|
|
|
181
202
|
finally:
|
|
182
203
|
runtime._active_connection.reset(token)
|
|
183
204
|
runtime._active_store.reset(store_token)
|
|
205
|
+
runtime._active_webhooks.reset(webhooks_token)
|
|
184
206
|
duration_ms = (time.perf_counter() - started) * 1000.0
|
|
185
207
|
self.call_log.append({"tool": tool_name, "match": "env", "input": supplied})
|
|
186
208
|
self.env._record_route(tool_name, "env", duration_ms, self.id)
|
|
@@ -234,6 +256,7 @@ class World:
|
|
|
234
256
|
self._refuse_a_changed_contract()
|
|
235
257
|
source, audit = self._seed()
|
|
236
258
|
self._relay(source, audit=audit)
|
|
259
|
+
self.webhooks.clear()
|
|
237
260
|
|
|
238
261
|
def _refuse_a_changed_contract(self) -> None:
|
|
239
262
|
"""An environment's bundle can change under a live world; a version's cannot."""
|
|
@@ -24,12 +24,16 @@ falls back to discovery by function name == served tool name.
|
|
|
24
24
|
from __future__ import annotations
|
|
25
25
|
|
|
26
26
|
import re
|
|
27
|
+
import sys
|
|
28
|
+
from contextlib import contextmanager
|
|
27
29
|
from contextvars import ContextVar
|
|
28
|
-
from
|
|
30
|
+
from types import ModuleType
|
|
31
|
+
from typing import Any, Callable, Dict, Iterator, Optional
|
|
29
32
|
|
|
33
|
+
from .schema.handler_errors import Refusal
|
|
30
34
|
from .schema.store_contract import STORE_METHOD_NAMES, unknown_store_method_detail
|
|
31
35
|
|
|
32
|
-
__all__ = ["tool", "db", "store"]
|
|
36
|
+
__all__ = ["tool", "db", "store", "webhooks", "Refusal"]
|
|
33
37
|
|
|
34
38
|
_active_connection: ContextVar[Optional[Any]] = ContextVar("gateway_env_active_connection", default=None)
|
|
35
39
|
|
|
@@ -38,6 +42,7 @@ _load_registry: ContextVar[Optional[Dict[str, Callable[..., Any]]]] = ContextVar
|
|
|
38
42
|
)
|
|
39
43
|
|
|
40
44
|
_active_store: ContextVar[Any] = ContextVar("gateway_env_active_store", default=None)
|
|
45
|
+
_active_webhooks: ContextVar[Any] = ContextVar("gateway_env_active_webhooks", default=None)
|
|
41
46
|
_strict_registration: ContextVar[bool] = ContextVar("gateway_env_strict_registration", default=False)
|
|
42
47
|
|
|
43
48
|
|
|
@@ -59,6 +64,21 @@ class _Store:
|
|
|
59
64
|
|
|
60
65
|
store = _Store()
|
|
61
66
|
|
|
67
|
+
|
|
68
|
+
class _Webhooks:
|
|
69
|
+
"""``deliver`` binds to the world whose tool is executing (:class:`~.schema.webhooks.WebhookOutbox`)."""
|
|
70
|
+
|
|
71
|
+
def deliver(
|
|
72
|
+
self, event_type: str, body: Any, scope: Optional[Dict[str, Any]] = None, event_id: Optional[str] = None
|
|
73
|
+
) -> list[str]:
|
|
74
|
+
active = _active_webhooks.get()
|
|
75
|
+
if active is None:
|
|
76
|
+
raise RuntimeError("gateway_env.webhooks.deliver requires an executing tool in a schema-backed world")
|
|
77
|
+
return active.deliver(event_type, body, scope, event_id)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
webhooks = _Webhooks()
|
|
81
|
+
|
|
62
82
|
_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
63
83
|
|
|
64
84
|
|
|
@@ -147,3 +167,26 @@ def tool(fn: Optional[Callable[..., Any]] = None, *, name: Optional[str] = None)
|
|
|
147
167
|
return register(handler, name or handler.__name__)
|
|
148
168
|
|
|
149
169
|
return decorator
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@contextmanager
|
|
173
|
+
def installed(module: ModuleType) -> Iterator[None]:
|
|
174
|
+
"""``gateway_env`` names ``module`` for a handler load's dynamic extent, and whatever it named before after."""
|
|
175
|
+
saved = sys.modules.get("gateway_env")
|
|
176
|
+
sys.modules["gateway_env"] = module
|
|
177
|
+
try:
|
|
178
|
+
yield
|
|
179
|
+
finally:
|
|
180
|
+
if saved is None:
|
|
181
|
+
sys.modules.pop("gateway_env", None)
|
|
182
|
+
else:
|
|
183
|
+
sys.modules["gateway_env"] = saved
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def route_module() -> ModuleType:
|
|
187
|
+
"""What a route handler may import: ``Refusal`` alone; ``store``, ``db`` and ``tool`` belong to tools."""
|
|
188
|
+
module = ModuleType("gateway_env")
|
|
189
|
+
module.Refusal = Refusal
|
|
190
|
+
module.__all__ = ["Refusal"]
|
|
191
|
+
return module
|
|
192
|
+
|
|
@@ -117,6 +117,7 @@ from .admin_routes import ADMIN, RUNNER_PATHS, AdminToken
|
|
|
117
117
|
from .bundle_runtime import stage_runtime_package
|
|
118
118
|
from .call_capture import Served, WorldRequestHandler, send_response
|
|
119
119
|
from .call_log import AnswerShape, CallLog, Via
|
|
120
|
+
from .webhooks import WebhookOutbox, Webhooks
|
|
120
121
|
from .compiler import check_bundle, compile_bundle
|
|
121
122
|
from .connector import SPEC, Operation, Spec, load_spec, stored_forms
|
|
122
123
|
from .cors import Cors
|
|
@@ -534,6 +535,7 @@ class WorldState:
|
|
|
534
535
|
self._dir = tempfile.TemporaryDirectory(prefix="gateway-world-api-")
|
|
535
536
|
self._lock = threading.RLock()
|
|
536
537
|
self.store: WorldDatabase | None = None
|
|
538
|
+
self.webhooks = WebhookOutbox(None, self.rows, self.key_of, ApiContext.clock)
|
|
537
539
|
self._clear_events()
|
|
538
540
|
self.reset()
|
|
539
541
|
|
|
@@ -557,8 +559,15 @@ class WorldState:
|
|
|
557
559
|
collide on every key)."""
|
|
558
560
|
with self._lock:
|
|
559
561
|
self._clear_events()
|
|
562
|
+
self.webhooks.clear()
|
|
560
563
|
self._open(self._snapshot_path if initial and self.snapshot else None, initial=initial)
|
|
561
564
|
|
|
565
|
+
def serve_webhooks(self, webhooks: Webhooks | None) -> None:
|
|
566
|
+
"""Deliveries this state's handlers stage go to the subscriptions ``webhooks`` declares."""
|
|
567
|
+
if webhooks is not None:
|
|
568
|
+
webhooks.check_contract(self.contract.entities)
|
|
569
|
+
self.webhooks = WebhookOutbox(webhooks, self.rows, self.key_of, ApiContext.clock)
|
|
570
|
+
|
|
562
571
|
def _clear_events(self) -> None:
|
|
563
572
|
self._events: list[dict] = []
|
|
564
573
|
self.event_seq = 0
|
|
@@ -708,12 +717,12 @@ class WorldState:
|
|
|
708
717
|
|
|
709
718
|
@contextmanager
|
|
710
719
|
def transaction(self) -> Iterator[None]:
|
|
711
|
-
"""One route call: its writes and
|
|
720
|
+
"""One route call: its writes, events and deliveries all land, or none do."""
|
|
712
721
|
with self._lock:
|
|
713
722
|
assert self.store is not None
|
|
714
723
|
events, seq = list(self._events), self.event_seq
|
|
715
724
|
try:
|
|
716
|
-
with self.store.transaction():
|
|
725
|
+
with self.webhooks.transaction(), self.store.transaction():
|
|
717
726
|
yield
|
|
718
727
|
except BaseException:
|
|
719
728
|
self._events, self.event_seq = events, seq
|
|
@@ -858,6 +867,13 @@ class ApiContext:
|
|
|
858
867
|
def events_since(self, seq: int) -> list[dict]:
|
|
859
868
|
return self.state.events_since(seq)
|
|
860
869
|
|
|
870
|
+
def deliver(
|
|
871
|
+
self, event_type: str, body: Any, scope: dict[str, Any] | None = None, event_id: str | None = None
|
|
872
|
+
) -> list[str]:
|
|
873
|
+
"""Send ``body`` as ``event_type`` to every subscription ``[webhooks]`` finds for it
|
|
874
|
+
(:class:`~.webhooks.WebhookOutbox`); the delivery ids, none when no one subscribed."""
|
|
875
|
+
return self.state.webhooks.deliver(event_type, body, scope, event_id)
|
|
876
|
+
|
|
861
877
|
def stored_forms(self, field: str, plaintext: Any) -> tuple[Any, ...]:
|
|
862
878
|
"""The values the store may hold for a plaintext under this world's ``[redact]``
|
|
863
879
|
(see :func:`connector.stored_forms`); a world without a spec stores the clear value."""
|
|
@@ -944,10 +960,13 @@ def load_handlers(settings: ApiSettings) -> dict[str, Handler]:
|
|
|
944
960
|
spec = importlib.util.spec_from_file_location(module_name, settings.handlers)
|
|
945
961
|
if spec is None or spec.loader is None:
|
|
946
962
|
fail("handlers", f"Could not load {settings.handlers}", "/api/handlers")
|
|
963
|
+
from .. import runtime
|
|
964
|
+
|
|
947
965
|
module = importlib.util.module_from_spec(spec)
|
|
948
966
|
sys.modules[module_name] = module
|
|
949
967
|
try:
|
|
950
|
-
|
|
968
|
+
with runtime.installed(runtime.route_module()):
|
|
969
|
+
spec.loader.exec_module(module)
|
|
951
970
|
finally:
|
|
952
971
|
sys.modules.pop(module_name, None)
|
|
953
972
|
handlers: dict[str, Handler] = {}
|
|
@@ -1035,6 +1054,7 @@ class WorldApp:
|
|
|
1035
1054
|
self.handlers = load_handlers(self.settings) if handlers is None else handlers
|
|
1036
1055
|
self.routes_closed = routes_closed
|
|
1037
1056
|
self.calls = calls if calls is not None else CallLog.for_spec(router.spec, secret=key)
|
|
1057
|
+
state.serve_webhooks(router.spec.webhooks)
|
|
1038
1058
|
|
|
1039
1059
|
def handle(
|
|
1040
1060
|
self,
|
|
@@ -1153,6 +1173,7 @@ class WorldApp:
|
|
|
1153
1173
|
self.calls.record(
|
|
1154
1174
|
operation, method, path, args, query_all, body, status, answer, elapsed, via, shape=shape, served=served
|
|
1155
1175
|
)
|
|
1176
|
+
self.calls.record_webhooks(self.state.webhooks.take())
|
|
1156
1177
|
return status, answer, settings.cors.carried(extra)
|
|
1157
1178
|
|
|
1158
1179
|
def _answer_atomically(self, op, operation, *request) -> Response:
|
|
@@ -36,11 +36,14 @@ from __future__ import annotations
|
|
|
36
36
|
import json
|
|
37
37
|
from dataclasses import dataclass, replace
|
|
38
38
|
from http.server import BaseHTTPRequestHandler
|
|
39
|
-
from typing import Any
|
|
39
|
+
from typing import TYPE_CHECKING, Any
|
|
40
40
|
|
|
41
41
|
from ..world_session_contract import REQUEST_BODY_CAPTURE_BYTES
|
|
42
42
|
from .call_redaction import SCRUB_MARGIN, CallRedaction
|
|
43
43
|
|
|
44
|
+
if TYPE_CHECKING:
|
|
45
|
+
from .webhooks import Delivery, WebhookAnswer
|
|
46
|
+
|
|
44
47
|
CAPTURE_BYTES = REQUEST_BODY_CAPTURE_BYTES
|
|
45
48
|
CONTENT_TYPE_CHARS = 256
|
|
46
49
|
JSON_TYPE = "application/json"
|
|
@@ -284,6 +287,15 @@ class Exchange:
|
|
|
284
287
|
"""A tool called by name: its arguments are what was asked, its answer what came back."""
|
|
285
288
|
return cls({}, CapturedBody.of_document(args, redaction), status, CapturedBody.of_document(answer, redaction))
|
|
286
289
|
|
|
290
|
+
@classmethod
|
|
291
|
+
def of_webhook(cls, delivery: Delivery, answer: WebhookAnswer, redaction: CallRedaction) -> Exchange:
|
|
292
|
+
"""A delivery the world sent: its signed headers and body, and the receiver's answer
|
|
293
|
+
(status 0 and no body until one answered; an error it never answered with as the body)."""
|
|
294
|
+
headers = redaction.headers(delivery.headers, redaction.secret, CAPTURE_BYTES)
|
|
295
|
+
error = None if answer.error is None else {"error": answer.error}
|
|
296
|
+
sent = CapturedBody.of_document(delivery.event.body, redaction)
|
|
297
|
+
return cls(headers, sent, answer.status, CapturedBody.of_document(error, redaction))
|
|
298
|
+
|
|
287
299
|
@classmethod
|
|
288
300
|
def from_json(cls, data: dict) -> Exchange:
|
|
289
301
|
request, response = data.get("request") or {}, data.get("response") or {}
|
|
@@ -21,7 +21,10 @@ from .connector import Operation, Spec, _dig
|
|
|
21
21
|
from .handler_errors import log_line
|
|
22
22
|
from .tasks import WITHHELD_FIELD
|
|
23
23
|
|
|
24
|
+
from .webhooks import ANSWER_SUFFIX, SUBSCRIPTIONS_SHOWN, WebhookAnswer, WebhookFeed, WebhookPoll
|
|
25
|
+
|
|
24
26
|
if TYPE_CHECKING:
|
|
27
|
+
from .webhooks import Delivery, WebhookOutbox
|
|
25
28
|
from .request_ledger import RequestOutbox
|
|
26
29
|
|
|
27
30
|
METHOD_CHARS = 32
|
|
@@ -129,6 +132,7 @@ class Via(StrEnum):
|
|
|
129
132
|
API = "api"
|
|
130
133
|
UI = "ui"
|
|
131
134
|
TOOL = "tool"
|
|
135
|
+
WEBHOOK = "webhook"
|
|
132
136
|
|
|
133
137
|
|
|
134
138
|
@dataclass(frozen=True)
|
|
@@ -302,6 +306,7 @@ class CallLog:
|
|
|
302
306
|
self.total = 0
|
|
303
307
|
self.journal = journal
|
|
304
308
|
self.outbox = outbox
|
|
309
|
+
self.webhooks = WebhookFeed()
|
|
305
310
|
self._lock = threading.Lock()
|
|
306
311
|
|
|
307
312
|
@classmethod
|
|
@@ -423,6 +428,47 @@ class CallLog:
|
|
|
423
428
|
request, budget = dict(WITHHELD_TOOL), RequestBudget()
|
|
424
429
|
self._append({**heading, **request, "via": str(Via.TOOL)}, answered, budget)
|
|
425
430
|
|
|
431
|
+
def record_webhooks(self, deliveries: Iterable[Delivery]) -> None:
|
|
432
|
+
"""Each delivery the world sent, kept by the request ledger with ``via`` webhook and no
|
|
433
|
+
answer yet (status 0); the grader's call log keeps only what the agent called."""
|
|
434
|
+
sent = list(deliveries)
|
|
435
|
+
self.webhooks.append(sent)
|
|
436
|
+
for delivery in sent:
|
|
437
|
+
self._send_webhook(delivery, WebhookAnswer(delivery.id, 0, None, None), {"delivery": delivery.id}, "")
|
|
438
|
+
|
|
439
|
+
def answer_webhooks(self, answers: Iterable[WebhookAnswer]) -> int:
|
|
440
|
+
"""Each receiver's answer to a forwarded delivery, kept by the request ledger as one more
|
|
441
|
+
``via`` webhook entry under ``<event type>.answer`` (so a check counting the event never counts
|
|
442
|
+
it) with the receiver's status and ``query.answer`` naming the delivery; the
|
|
443
|
+
first answer to each delivery only, and none for an id the feed no longer holds."""
|
|
444
|
+
kept = 0
|
|
445
|
+
for answer in answers:
|
|
446
|
+
delivery = self.webhooks.answer(answer.delivery_id)
|
|
447
|
+
if delivery is None:
|
|
448
|
+
continue
|
|
449
|
+
self._send_webhook(delivery, answer, {"answer": delivery.id}, ANSWER_SUFFIX)
|
|
450
|
+
kept += 1
|
|
451
|
+
return kept
|
|
452
|
+
|
|
453
|
+
def webhook_page(self, outbox: WebhookOutbox | None, poll: WebhookPoll) -> dict:
|
|
454
|
+
"""A forwarder's poll: its answers kept first, then the subscriptions ``outbox`` reads (none
|
|
455
|
+
declared: ``declared`` false; at most :data:`SUBSCRIPTIONS_SHOWN`, each with how many deliveries
|
|
456
|
+
this feed sent it) and the page of deliveries."""
|
|
457
|
+
declared = outbox is not None and outbox.webhooks is not None
|
|
458
|
+
kept = self.answer_webhooks(poll.answers)
|
|
459
|
+
found = outbox.subscriptions() if outbox is not None and poll.subscriptions else []
|
|
460
|
+
shown = [{**sub, "sent": self.webhooks.sent_to(sub["key"])} for sub in found[:SUBSCRIPTIONS_SHOWN]]
|
|
461
|
+
page = self.webhooks.after(poll.after, poll.limit, poll.epoch, poll.tail, poll.targets)
|
|
462
|
+
listed = {"subscriptions": shown, "truncated": len(found) > len(shown)}
|
|
463
|
+
return {"declared": declared, **listed, "answered": kept, **page}
|
|
464
|
+
|
|
465
|
+
def _send_webhook(self, delivery: Delivery, answer: WebhookAnswer, query: dict, suffix: str) -> None:
|
|
466
|
+
path = self.redaction.raw_path(delivery.endpoint, self.redaction.secret)
|
|
467
|
+
line = {"operation": f"{delivery.event.type}{suffix}", "method": "POST", "path": path, "query": query}
|
|
468
|
+
exchange = partial(Exchange.of_webhook, delivery, answer, self.redaction)
|
|
469
|
+
answered = _answered(answer.status, None, (answer.duration_ms or 0.0) / 1000)
|
|
470
|
+
self._send({**line, "via": str(Via.WEBHOOK)}, answered, exchange)
|
|
471
|
+
|
|
426
472
|
def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
|
|
427
473
|
budget = RequestBudget()
|
|
428
474
|
logged_args = self.redaction.value(args, budget)
|
|
@@ -179,8 +179,14 @@ KEPT_HEADERS = frozenset(
|
|
|
179
179
|
"content-type",
|
|
180
180
|
"host",
|
|
181
181
|
"idempotency-key",
|
|
182
|
+
"linear-delivery",
|
|
183
|
+
"linear-event",
|
|
182
184
|
"origin",
|
|
185
|
+
"svix-id",
|
|
186
|
+
"svix-timestamp",
|
|
183
187
|
"user-agent",
|
|
188
|
+
"x-github-delivery",
|
|
189
|
+
"x-github-event",
|
|
184
190
|
"x-forwarded-proto",
|
|
185
191
|
"x-request-id",
|
|
186
192
|
}
|
|
@@ -899,7 +899,10 @@ def bind_connection(bundle: Path, contract: Contract) -> None:
|
|
|
899
899
|
return
|
|
900
900
|
from .connector import parse_spec
|
|
901
901
|
|
|
902
|
-
parse_spec(spec_path.read_text(encoding="utf-8"))
|
|
902
|
+
spec = parse_spec(spec_path.read_text(encoding="utf-8"))
|
|
903
|
+
spec.check_entities(contract.entities)
|
|
904
|
+
if spec.webhooks is not None:
|
|
905
|
+
spec.webhooks.check_contract(contract.entities)
|
|
903
906
|
|
|
904
907
|
|
|
905
908
|
def check_bundle(bundle: str | Path) -> Contract:
|
|
@@ -44,6 +44,9 @@ from .validation import fail
|
|
|
44
44
|
PIN = "conform"
|
|
45
45
|
|
|
46
46
|
|
|
47
|
+
BODILESS_STATUSES = (204, 205)
|
|
48
|
+
|
|
49
|
+
|
|
47
50
|
def load_openapi(path: str | Path) -> dict:
|
|
48
51
|
return parse_openapi(Path(path).read_text(encoding="utf-8"), str(path))
|
|
49
52
|
|
|
@@ -316,7 +319,7 @@ class Outcome:
|
|
|
316
319
|
self.documented
|
|
317
320
|
and self.status is not None
|
|
318
321
|
and str(self.status) in self.expected
|
|
319
|
-
and self.schema in ("ok", "no-schema")
|
|
322
|
+
and self.schema in ("ok", "no-schema", "no-body")
|
|
320
323
|
)
|
|
321
324
|
|
|
322
325
|
|
|
@@ -399,13 +402,28 @@ def _first_value(rows: list[dict], path: str) -> Any:
|
|
|
399
402
|
return None
|
|
400
403
|
|
|
401
404
|
|
|
405
|
+
def _keyed_value(state: WorldState, name: str) -> Any:
|
|
406
|
+
for entity, declaration in state.contract.entities.items():
|
|
407
|
+
if name in declaration["primary_key"]:
|
|
408
|
+
value = _first_value(state.rows(entity), name)
|
|
409
|
+
if value is not None:
|
|
410
|
+
return value
|
|
411
|
+
return None
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _own_key_value(state: WorldState, entity: str) -> Any:
|
|
415
|
+
key = state.contract.entities[entity]["primary_key"]
|
|
416
|
+
return _first_value(state.rows(entity), key[0]) if len(key) == 1 else None
|
|
417
|
+
|
|
418
|
+
|
|
402
419
|
def _sample_args(
|
|
403
420
|
operation: Operation, spec, settings, state: WorldState, checker: Checker, documented: dict | None
|
|
404
421
|
) -> tuple[dict[str, str], dict[str, str]]:
|
|
405
422
|
"""Path arguments from the world's own rows — the operation's declared ``sample``
|
|
406
423
|
first (a value a validating handler accepts), then this operation's filter, then any
|
|
407
|
-
operation's filter on the same placeholder, then
|
|
408
|
-
|
|
424
|
+
operation's filter on the same placeholder, then a row of this operation's entity holding a field of that
|
|
425
|
+
name, then the first row of any entity keyed by it, then (for the path's last placeholder only) the entity's
|
|
426
|
+
primary key — else the document's examples. The point is to hit data, not to probe 404s."""
|
|
409
427
|
op = settings.for_operation(operation.name)
|
|
410
428
|
hints = _hints(spec, settings)
|
|
411
429
|
args: dict[str, str] = {}
|
|
@@ -422,9 +440,11 @@ def _sample_args(
|
|
|
422
440
|
entity, path = hints[name]
|
|
423
441
|
value = _first_value(state.rows(entity), path)
|
|
424
442
|
if value is None and operation.entity:
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
443
|
+
value = _first_value(state.rows(operation.entity), name)
|
|
444
|
+
if value is None:
|
|
445
|
+
value = _keyed_value(state, name)
|
|
446
|
+
if value is None and operation.entity and name == list(operation.placeholders)[-1]:
|
|
447
|
+
value = _own_key_value(state, operation.entity)
|
|
428
448
|
if value is None and name in documented_params:
|
|
429
449
|
value = checker.example(documented_params[name].get("schema") or {})
|
|
430
450
|
args[name] = str(value if value is not None else "1")
|
|
@@ -448,7 +468,9 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
|
|
|
448
468
|
outcomes: list[Outcome] = []
|
|
449
469
|
try:
|
|
450
470
|
door = Door.of(spec, settings, PIN)
|
|
451
|
-
|
|
471
|
+
operations = list(spec.operations.values())[:max_operations]
|
|
472
|
+
operations.sort(key=lambda operation: operation.method == "DELETE")
|
|
473
|
+
for operation in operations:
|
|
452
474
|
located = None
|
|
453
475
|
checker = None
|
|
454
476
|
for document in documents:
|
|
@@ -495,7 +517,9 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
|
|
|
495
517
|
outcome.status = status
|
|
496
518
|
answer = responses.get(str(status)) or responses.get("default") or {}
|
|
497
519
|
schema = ((answer.get("content") or {}).get("application/json") or {}).get("schema")
|
|
498
|
-
if
|
|
520
|
+
if status in BODILESS_STATUSES and not raw:
|
|
521
|
+
outcome.schema = "no-body"
|
|
522
|
+
elif schema is None:
|
|
499
523
|
outcome.schema = "no-schema"
|
|
500
524
|
else:
|
|
501
525
|
try:
|
|
@@ -93,6 +93,7 @@ from .validation import (
|
|
|
93
93
|
string_of,
|
|
94
94
|
table_of,
|
|
95
95
|
)
|
|
96
|
+
from .webhooks import Webhooks
|
|
96
97
|
|
|
97
98
|
SPEC = "connector.toml"
|
|
98
99
|
CAPTURES = "captures"
|
|
@@ -175,6 +176,7 @@ class Spec:
|
|
|
175
176
|
round_fields: dict[str, int] = field(default_factory=dict)
|
|
176
177
|
preserve_fields: frozenset[str] = frozenset()
|
|
177
178
|
operations: dict[str, Operation] = field(default_factory=dict)
|
|
179
|
+
webhooks: Webhooks | None = None
|
|
178
180
|
|
|
179
181
|
@property
|
|
180
182
|
def scheme(self) -> str:
|
|
@@ -208,6 +210,8 @@ class Spec:
|
|
|
208
210
|
"`gateway worlds schema entity add`, or point the route at one that exists",
|
|
209
211
|
location,
|
|
210
212
|
)
|
|
213
|
+
if self.webhooks is not None:
|
|
214
|
+
self.webhooks.check_entity(known)
|
|
211
215
|
|
|
212
216
|
|
|
213
217
|
RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
|
|
@@ -343,6 +347,7 @@ def spec_from(raw: dict) -> Spec:
|
|
|
343
347
|
round_fields=round_fields,
|
|
344
348
|
preserve_fields=preserve_fields,
|
|
345
349
|
operations=operations,
|
|
350
|
+
webhooks=Webhooks.parse(raw.get("webhooks")),
|
|
346
351
|
)
|
|
347
352
|
|
|
348
353
|
|
|
@@ -33,7 +33,8 @@ hosted call into such a handler answers 400 {error: "handler", detail, handler}:
|
|
|
33
33
|
Rows per entity, as {entity: count}.
|
|
34
34
|
|
|
35
35
|
A handler that raises AttributeError, TypeError or NameError is answered 400 with
|
|
36
|
-
that structured body;
|
|
36
|
+
that structured body; `raise Refusal(404, body)` (from gateway_env) answers that 4xx
|
|
37
|
+
and body as the vendor's refusal; any other exception is answered 500 {error: "engine", ...}.
|
|
37
38
|
`schema init --connector custom --with-example-tool` writes the example in full.
|
|
38
39
|
"""
|
|
39
40
|
|