@withgateway/sdk 0.4.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +573 -0
- package/dist/agents/index.d.ts +573 -0
- package/dist/agents/index.js +1443 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1394 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +5655 -1890
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +5769 -2005
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +418 -120
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +418 -120
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-DUHmS74p.d.ts → index-DIN14Lhg.d.ts} +27 -4
- package/dist/{index-mJhG3vZU.d.mts → index-DuB-eQ0Z.d.mts} +27 -4
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +2862 -438
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2820 -431
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +418 -120
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +418 -120
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +2086 -1749
- package/dist/worlds/index.d.ts +2086 -1749
- package/dist/worlds/index.js +1471 -269
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1466 -265
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_world.py +24 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +243 -0
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/api.py +27 -7
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +2 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +13 -1
- package/runtime/gatewaysdk/environment/schema/call_log.py +66 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +6 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +32 -8
- package/runtime/gatewaysdk/environment/schema/connector.py +5 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +9 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/database/guard.py +2 -1
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +27 -1
- package/runtime/gatewaysdk/environment/schema/host.py +74 -11
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +33 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_requests.py +14 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +13 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +96 -19
- package/runtime/gatewaysdk/environment/schema/journal.py +10 -4
- package/runtime/gatewaysdk/environment/schema/local.py +33 -7
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +33 -480
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +332 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +49 -4
- package/runtime/gatewaysdk/environment/schema/scaffold.py +369 -218
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +20 -24
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +45 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +40 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +30 -4
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +26 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +10 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +6 -0
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +5 -1
- package/runtime/gatewaysdk/environment/schema/workers.py +8 -1
- package/runtime/gatewaysdk/environment/schema/world_host.py +64 -18
- package/runtime/gatewaysdk/environment/schema/world_tests.py +74 -0
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +20 -24
- package/skills/connector-schema-authoring/connector-worlds.md +155 -13
- package/skills/connector-schema-authoring/world-schema.md +36 -29
- package/skills/world-data-ingestion/SKILL.md +11 -11
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +45 -12
- package/skills/worlds-getting-started/references/gotchas.md +40 -9
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -0,0 +1,687 @@
|
|
|
1
|
+
"""Webhooks a world sends, declared once in ``connector.toml``::
|
|
2
|
+
|
|
3
|
+
[webhooks]
|
|
4
|
+
subscriptions = "webhooks" # the entity agents register endpoints in
|
|
5
|
+
url = "url" # that entity's fields
|
|
6
|
+
events = "event_types" # absent or empty: every event
|
|
7
|
+
secret = "secret"
|
|
8
|
+
enabled = "enabled" # absent: always on; false: off
|
|
9
|
+
scope = { inbox_id = "inbox_ids" } # a scope value reaches a row whose list holds it (or is empty)
|
|
10
|
+
signing = "svix" # svix | github | linear | stripe | hmac-sha256 | none
|
|
11
|
+
signature_header = "x-signature" # hmac-sha256 only
|
|
12
|
+
|
|
13
|
+
A handler calls ``ctx.deliver(event_type, body, scope=...)`` (a route) or
|
|
14
|
+
``gateway_env.webhooks.deliver(...)`` (a tool). Each enabled subscription that wants the
|
|
15
|
+
event and holds its scope gets one signed :class:`Delivery`, kept only when the call
|
|
16
|
+
that staged it commits (:class:`WebhookOutbox`)."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import base64
|
|
21
|
+
import binascii
|
|
22
|
+
import hashlib
|
|
23
|
+
import hmac
|
|
24
|
+
import json
|
|
25
|
+
import re
|
|
26
|
+
import threading
|
|
27
|
+
import urllib.parse
|
|
28
|
+
import uuid
|
|
29
|
+
from abc import ABC, abstractmethod
|
|
30
|
+
from collections.abc import Callable, Iterable, Iterator, Mapping
|
|
31
|
+
from contextlib import contextmanager
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from typing import Any, ClassVar
|
|
34
|
+
|
|
35
|
+
from .handler_errors import log_line
|
|
36
|
+
from .validation import fail
|
|
37
|
+
|
|
38
|
+
LOCATION = "/webhooks"
|
|
39
|
+
HELD_DELIVERIES = 1000
|
|
40
|
+
MAX_DELIVERIES_PER_EVENT = 100
|
|
41
|
+
MAX_DELIVERIES_PER_CALL = 500
|
|
42
|
+
RECEIVER_SCHEMES = frozenset({"http", "https"})
|
|
43
|
+
MAX_PAYLOAD_BYTES = 256 << 10
|
|
44
|
+
EVENT_ID = re.compile(r"[A-Za-z0-9_.:-]{1,128}")
|
|
45
|
+
EVENT_TYPE = EVENT_ID
|
|
46
|
+
FEED_DELIVERIES = 500
|
|
47
|
+
FEED_BYTES = 8 << 20
|
|
48
|
+
PAGE_BYTES = 4 << 20
|
|
49
|
+
SUBSCRIPTIONS_SHOWN = 200
|
|
50
|
+
ANSWER_SUFFIX = ".answer"
|
|
51
|
+
FEED_EPOCH = re.compile(r"[0-9a-f]{32}")
|
|
52
|
+
FEED_PAGE_MAX = 100
|
|
53
|
+
JSON_TYPE = "application/json"
|
|
54
|
+
SVIX_PREFIX = "whsec_"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class WebhookMisuse(TypeError):
|
|
58
|
+
"""A handler's call to deliver its author got wrong: answered as the handler's bug, never as
|
|
59
|
+
the contract refusing the agent's request."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Unsignable(ValueError):
|
|
63
|
+
"""A subscription its scheme cannot sign for: skipped, the rest still delivered."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class WebhookEvent:
|
|
68
|
+
"""``payload`` is the body as signed, frozen when the handler sent it; ``body`` is that payload read
|
|
69
|
+
back, so what the ledger keeps and a forwarder replays is what was signed, whatever the handler
|
|
70
|
+
does to its own object afterwards."""
|
|
71
|
+
|
|
72
|
+
type: str
|
|
73
|
+
body: Any
|
|
74
|
+
scope: Mapping[str, str]
|
|
75
|
+
id: str
|
|
76
|
+
payload: bytes
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def sent(cls, event_type: str, body: Any, scope: Mapping[str, str], event_id: str) -> WebhookEvent:
|
|
80
|
+
payload = _frozen(body)
|
|
81
|
+
if len(payload) > MAX_PAYLOAD_BYTES:
|
|
82
|
+
raise WebhookMisuse(f"a webhook body is at most {MAX_PAYLOAD_BYTES} bytes as JSON")
|
|
83
|
+
return cls(event_type, json.loads(payload), scope, event_id, payload)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _string_keys(value: Any) -> bool:
|
|
87
|
+
stack = [value]
|
|
88
|
+
while stack:
|
|
89
|
+
item = stack.pop()
|
|
90
|
+
if isinstance(item, dict):
|
|
91
|
+
if not all(isinstance(key, str) for key in item):
|
|
92
|
+
return False
|
|
93
|
+
stack.extend(item.values())
|
|
94
|
+
if isinstance(item, (list, tuple)):
|
|
95
|
+
stack.extend(item)
|
|
96
|
+
return True
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _frozen(body: Any) -> bytes:
|
|
100
|
+
if not _string_keys(body):
|
|
101
|
+
raise WebhookMisuse("a webhook body's object keys are strings")
|
|
102
|
+
try:
|
|
103
|
+
text = json.dumps(body, separators=(",", ":"), ensure_ascii=False, allow_nan=False)
|
|
104
|
+
except (TypeError, ValueError) as error:
|
|
105
|
+
raise WebhookMisuse(f"a webhook body is JSON: {error}") from None
|
|
106
|
+
try:
|
|
107
|
+
return text.encode("utf-8")
|
|
108
|
+
except UnicodeEncodeError:
|
|
109
|
+
fail("contract", "a webhook body holds an unpaired Unicode surrogate", LOCATION)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(frozen=True)
|
|
113
|
+
class Delivery:
|
|
114
|
+
"""One event signed for one subscription, as the receiver will get it."""
|
|
115
|
+
|
|
116
|
+
id: str
|
|
117
|
+
event: WebhookEvent
|
|
118
|
+
subscription: dict
|
|
119
|
+
url: str
|
|
120
|
+
headers: dict[str, str]
|
|
121
|
+
payload: bytes
|
|
122
|
+
at: int
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def endpoint(self) -> str:
|
|
126
|
+
"""The receiver's URL without its userinfo, query or fragment, each of which may carry the
|
|
127
|
+
receiver's own credential."""
|
|
128
|
+
return _public_url(self.url)
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def size(self) -> int:
|
|
132
|
+
return len(self.payload) + sum(len(name) + len(value) for name, value in self.headers.items())
|
|
133
|
+
|
|
134
|
+
def as_json(self, target: bool = False) -> dict:
|
|
135
|
+
"""What a forwarder replays: ``payload`` is the exact text the headers sign; ``target``, the
|
|
136
|
+
receiver's whole URL for the platform's push relay alone, which posts to it."""
|
|
137
|
+
whole = {"target": self.url} if target else {}
|
|
138
|
+
return {
|
|
139
|
+
**whole,
|
|
140
|
+
"id": self.id,
|
|
141
|
+
"eventId": self.event.id,
|
|
142
|
+
"eventType": self.event.type,
|
|
143
|
+
"subscription": self.subscription,
|
|
144
|
+
"url": self.endpoint,
|
|
145
|
+
"headers": self.headers,
|
|
146
|
+
"payload": self.payload.decode("utf-8"),
|
|
147
|
+
"at": self.at,
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class Signer(ABC):
|
|
152
|
+
"""How a vendor signs what it sends: the headers one delivery carries."""
|
|
153
|
+
|
|
154
|
+
name: ClassVar[str]
|
|
155
|
+
needs_secret: ClassVar[bool] = True
|
|
156
|
+
signs_without_secret: ClassVar[bool] = False
|
|
157
|
+
|
|
158
|
+
@abstractmethod
|
|
159
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]: ...
|
|
160
|
+
|
|
161
|
+
@staticmethod
|
|
162
|
+
def _hex(key: bytes, payload: bytes) -> str:
|
|
163
|
+
return hmac.new(key, payload, hashlib.sha256).hexdigest()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class SvixSigner(Signer):
|
|
167
|
+
"""Svix (AgentMail, Resend, Clerk): ``v1,`` + base64 HMAC-SHA256 of ``id.timestamp.body``
|
|
168
|
+
under the ``whsec_`` secret's base64 key."""
|
|
169
|
+
|
|
170
|
+
name = "svix"
|
|
171
|
+
|
|
172
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]:
|
|
173
|
+
signed = f"{event.id}.{at}.".encode() + payload
|
|
174
|
+
digest = hmac.new(self._key(secret), signed, hashlib.sha256).digest()
|
|
175
|
+
return {
|
|
176
|
+
"svix-id": event.id,
|
|
177
|
+
"svix-timestamp": str(at),
|
|
178
|
+
"svix-signature": "v1," + base64.b64encode(digest).decode(),
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
@staticmethod
|
|
182
|
+
def _key(secret: str) -> bytes:
|
|
183
|
+
try:
|
|
184
|
+
key = base64.b64decode(secret.removeprefix(SVIX_PREFIX), validate=True)
|
|
185
|
+
except (binascii.Error, ValueError):
|
|
186
|
+
raise Unsignable(f"a svix secret is {SVIX_PREFIX} and base64") from None
|
|
187
|
+
if not key:
|
|
188
|
+
raise Unsignable("a svix secret holds a key")
|
|
189
|
+
return key
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class GithubSigner(Signer):
|
|
193
|
+
"""GitHub signs only a hook that has a secret; one without is sent unsigned."""
|
|
194
|
+
|
|
195
|
+
name = "github"
|
|
196
|
+
signs_without_secret = True
|
|
197
|
+
|
|
198
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]:
|
|
199
|
+
named = {"x-github-event": event.type, "x-github-delivery": delivery_id}
|
|
200
|
+
if not secret:
|
|
201
|
+
return named
|
|
202
|
+
return {**named, "x-hub-signature-256": "sha256=" + self._hex(secret.encode(), payload)}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class LinearSigner(Signer):
|
|
206
|
+
"""Linear: the bare hex HMAC-SHA256 of the body in ``linear-signature``."""
|
|
207
|
+
|
|
208
|
+
name = "linear"
|
|
209
|
+
|
|
210
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]:
|
|
211
|
+
return {
|
|
212
|
+
"linear-delivery": delivery_id,
|
|
213
|
+
"linear-event": event.type,
|
|
214
|
+
"linear-signature": self._hex(secret.encode(), payload),
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class StripeSigner(Signer):
|
|
219
|
+
"""Stripe: ``t=<timestamp>,v1=<hex HMAC-SHA256 of timestamp.body>`` in ``stripe-signature``."""
|
|
220
|
+
|
|
221
|
+
name = "stripe"
|
|
222
|
+
|
|
223
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]:
|
|
224
|
+
signed = f"{at}.".encode() + payload
|
|
225
|
+
return {"stripe-signature": f"t={at},v1={self._hex(secret.encode(), signed)}"}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class HmacSigner(Signer):
|
|
229
|
+
name = "hmac-sha256"
|
|
230
|
+
|
|
231
|
+
def __init__(self, header: str) -> None:
|
|
232
|
+
self.header = header
|
|
233
|
+
|
|
234
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]:
|
|
235
|
+
return {self.header: "sha256=" + self._hex(secret.encode(), payload)}
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class UnsignedSigner(Signer):
|
|
239
|
+
name = "none"
|
|
240
|
+
needs_secret = False
|
|
241
|
+
|
|
242
|
+
def headers(self, delivery_id: str, event: WebhookEvent, payload: bytes, secret: str, at: int) -> dict[str, str]:
|
|
243
|
+
return {}
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
SIGNERS: dict[str, Callable[[dict], Signer]] = {
|
|
247
|
+
SvixSigner.name: lambda table: SvixSigner(),
|
|
248
|
+
GithubSigner.name: lambda table: GithubSigner(),
|
|
249
|
+
LinearSigner.name: lambda table: LinearSigner(),
|
|
250
|
+
StripeSigner.name: lambda table: StripeSigner(),
|
|
251
|
+
HmacSigner.name: lambda table: HmacSigner(_field(table, "signature_header", "x-webhook-signature").lower()),
|
|
252
|
+
UnsignedSigner.name: lambda table: UnsignedSigner(),
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _public_url(url: str) -> str:
|
|
257
|
+
parts = urllib.parse.urlsplit(url)
|
|
258
|
+
host = parts.hostname or ""
|
|
259
|
+
host = f"[{host}]" if ":" in host else host
|
|
260
|
+
port = _port(parts)
|
|
261
|
+
return urllib.parse.urlunsplit((parts.scheme, f"{host}{port}", parts.path, "", ""))
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _port(parts: urllib.parse.SplitResult) -> str:
|
|
265
|
+
try:
|
|
266
|
+
return "" if parts.port is None else f":{parts.port}"
|
|
267
|
+
except ValueError:
|
|
268
|
+
return ""
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _items(value: Any) -> list[str] | None:
|
|
272
|
+
"""A field's values as text: absent or empty is ``[]`` (every value), one string or integer its one value
|
|
273
|
+
(Jamf's ``event``, Linear's ``teamId``, GitHub's ``repo_id``), anything else that is not a list None (none)."""
|
|
274
|
+
if isinstance(value, bool):
|
|
275
|
+
return None
|
|
276
|
+
if value is None or value in ("", [], {}):
|
|
277
|
+
return []
|
|
278
|
+
if isinstance(value, (str, int)):
|
|
279
|
+
return [str(value)]
|
|
280
|
+
return [str(item) for item in value] if isinstance(value, list) else None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _shown(value: Any) -> list[str]:
|
|
284
|
+
items = _items(value)
|
|
285
|
+
return [str(value)] if items is None else items
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _admits(listed: Any, value: str) -> bool:
|
|
289
|
+
"""An absent or empty list admits every value; any other list only its members, compared as text."""
|
|
290
|
+
items = _items(listed)
|
|
291
|
+
return items == [] or (items is not None and value in items)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _receivable(url: Any) -> bool:
|
|
295
|
+
"""An http(s) URL with a host and a port that parses: anything else no receiver could be at."""
|
|
296
|
+
if not isinstance(url, str) or not url:
|
|
297
|
+
return False
|
|
298
|
+
try:
|
|
299
|
+
parts = urllib.parse.urlsplit(url)
|
|
300
|
+
_ = parts.port
|
|
301
|
+
except ValueError:
|
|
302
|
+
return False
|
|
303
|
+
bracketed = parts.netloc.rsplit("@", 1)[-1].startswith("[")
|
|
304
|
+
host = parts.hostname or ""
|
|
305
|
+
return parts.scheme in RECEIVER_SCHEMES and bool(host) and (":" in host or not bracketed)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _field(table: dict, key: str, default: str | None = None) -> str | None:
|
|
309
|
+
value = table.get(key, default)
|
|
310
|
+
if value is not None and (not isinstance(value, str) or not value):
|
|
311
|
+
fail("spec", f"[webhooks] {key} names a field: a non-empty string", f"{LOCATION}/{key}")
|
|
312
|
+
return value
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _scope(table: dict) -> dict[str, str]:
|
|
316
|
+
scope = table.get("scope", {})
|
|
317
|
+
if not isinstance(scope, dict) or not all(isinstance(v, str) and v for v in scope.values()):
|
|
318
|
+
fail("spec", "[webhooks] scope maps an event's scope key to a subscription field", f"{LOCATION}/scope")
|
|
319
|
+
return dict(scope)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@dataclass(frozen=True)
|
|
323
|
+
class Webhooks:
|
|
324
|
+
"""The ``[webhooks]`` table: where subscriptions live and how deliveries are signed."""
|
|
325
|
+
|
|
326
|
+
subscriptions: str
|
|
327
|
+
url: str
|
|
328
|
+
signer: Signer
|
|
329
|
+
events: str | None = None
|
|
330
|
+
secret: str | None = None
|
|
331
|
+
enabled: str | None = None
|
|
332
|
+
scope: dict[str, str] = field(default_factory=dict)
|
|
333
|
+
|
|
334
|
+
@classmethod
|
|
335
|
+
def parse(cls, raw: Any) -> Webhooks | None:
|
|
336
|
+
if raw is None:
|
|
337
|
+
return None
|
|
338
|
+
if not isinstance(raw, dict):
|
|
339
|
+
fail("spec", "[webhooks] is a table", LOCATION)
|
|
340
|
+
signing = raw.get("signing", "none")
|
|
341
|
+
if signing not in SIGNERS:
|
|
342
|
+
fail("spec", f"[webhooks] signing is one of {', '.join(SIGNERS)}", f"{LOCATION}/signing")
|
|
343
|
+
if "signature_header" in raw and signing != HmacSigner.name:
|
|
344
|
+
fail("spec", "[webhooks] signature_header belongs to hmac-sha256 signing", f"{LOCATION}/signature_header")
|
|
345
|
+
webhooks = cls(
|
|
346
|
+
subscriptions=_field(raw, "subscriptions") or fail("spec", "[webhooks] names its subscriptions entity", LOCATION),
|
|
347
|
+
url=_field(raw, "url", "url"), # type: ignore[arg-type]
|
|
348
|
+
signer=SIGNERS[signing](raw),
|
|
349
|
+
events=_field(raw, "events"),
|
|
350
|
+
secret=_field(raw, "secret"),
|
|
351
|
+
enabled=_field(raw, "enabled"),
|
|
352
|
+
scope=_scope(raw),
|
|
353
|
+
)
|
|
354
|
+
if webhooks.signer.needs_secret and webhooks.secret is None:
|
|
355
|
+
fail("spec", f"[webhooks] signing {signing} needs the secret field", f"{LOCATION}/secret")
|
|
356
|
+
return webhooks
|
|
357
|
+
|
|
358
|
+
@property
|
|
359
|
+
def fields(self) -> dict[str, str]:
|
|
360
|
+
named = {"url": self.url, "events": self.events, "secret": self.secret, "enabled": self.enabled}
|
|
361
|
+
scoped = {f"scope/{key}": value for key, value in self.scope.items()}
|
|
362
|
+
return {key: value for key, value in {**named, **scoped}.items() if value is not None}
|
|
363
|
+
|
|
364
|
+
def check_entity(self, declared: Iterable[str]) -> None:
|
|
365
|
+
known = set(declared)
|
|
366
|
+
if self.subscriptions not in known:
|
|
367
|
+
fail(
|
|
368
|
+
"spec",
|
|
369
|
+
f"[webhooks] subscriptions names {self.subscriptions}, which schema/world.json does not declare",
|
|
370
|
+
f"{LOCATION}/subscriptions",
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
def check_contract(self, entities: Mapping[str, dict]) -> None:
|
|
374
|
+
"""The subscriptions entity is declared and has every field the table names."""
|
|
375
|
+
self.check_entity(entities)
|
|
376
|
+
schema = entities[self.subscriptions].get("schema") or {}
|
|
377
|
+
known = set(schema.get("properties") or {})
|
|
378
|
+
missing = {key: name for key, name in self.fields.items() if name not in known}
|
|
379
|
+
if missing:
|
|
380
|
+
key, name = next(iter(missing.items()))
|
|
381
|
+
fail("spec", f"[webhooks] {key} names {name}, which {self.subscriptions} does not have", f"{LOCATION}/{key}")
|
|
382
|
+
|
|
383
|
+
def event(self, event_type: Any, body: Any, scope: Any, event_id: Any) -> WebhookEvent:
|
|
384
|
+
if not (isinstance(event_type, str) and EVENT_TYPE.fullmatch(event_type)):
|
|
385
|
+
raise WebhookMisuse("an event type is 1 to 128 letters, digits and _ . : -")
|
|
386
|
+
if event_type.endswith(ANSWER_SUFFIX):
|
|
387
|
+
raise WebhookMisuse(f"an event type never ends in {ANSWER_SUFFIX}: the ledger names receivers' answers so")
|
|
388
|
+
if event_id is not None and not (isinstance(event_id, str) and EVENT_ID.fullmatch(event_id)):
|
|
389
|
+
raise WebhookMisuse("an event_id is 1 to 128 letters, digits and _ . : -")
|
|
390
|
+
scope = dict(scope or {})
|
|
391
|
+
unknown = sorted(set(scope) - set(self.scope))
|
|
392
|
+
if unknown:
|
|
393
|
+
raise WebhookMisuse(f"deliver's scope {', '.join(unknown)} is not in [webhooks] scope")
|
|
394
|
+
scoped = {k: str(v) for k, v in scope.items()}
|
|
395
|
+
event = WebhookEvent.sent(event_type, body, scoped, event_id or f"msg_{uuid.uuid4().hex}")
|
|
396
|
+
return event
|
|
397
|
+
|
|
398
|
+
def deliveries(self, rows: Iterable[dict], key: Callable[[dict], dict], event: WebhookEvent, at: int) -> list[Delivery]:
|
|
399
|
+
"""One delivery per subscription that receives ``event``, at most :data:`MAX_DELIVERIES_PER_EVENT`;
|
|
400
|
+
a subscription its scheme cannot sign for, and any past the bound, is skipped with a log line."""
|
|
401
|
+
payload = event.payload
|
|
402
|
+
receiving = [row for row in rows if self.receives(row, event)]
|
|
403
|
+
signed = [self._signed(row, key(row), event, payload, at) for row in receiving[:MAX_DELIVERIES_PER_EVENT]]
|
|
404
|
+
found = [delivery for delivery in signed if delivery is not None]
|
|
405
|
+
skipped = len(receiving) - len(found)
|
|
406
|
+
if skipped:
|
|
407
|
+
log_line(f"webhooks: {event.type} skipped {skipped} subscription(s), unsignable or past the bound")
|
|
408
|
+
return found
|
|
409
|
+
|
|
410
|
+
def _signed(self, row: dict, key: dict, event: WebhookEvent, payload: bytes, at: int) -> Delivery | None:
|
|
411
|
+
try:
|
|
412
|
+
return self._delivery(row, key, event, payload, at)
|
|
413
|
+
except Unsignable:
|
|
414
|
+
return None
|
|
415
|
+
|
|
416
|
+
def receives(self, row: dict, event: WebhookEvent) -> bool:
|
|
417
|
+
return _receivable(row.get(self.url)) and self._enabled(row) and self._wants(row, event.type) and self._holds(row, event.scope)
|
|
418
|
+
|
|
419
|
+
def _enabled(self, row: dict) -> bool:
|
|
420
|
+
return self.enabled is None or row.get(self.enabled) is not False
|
|
421
|
+
|
|
422
|
+
def subscription(self, row: dict, key: dict) -> dict:
|
|
423
|
+
"""One subscription as a reader sees it: never its secret, its url without the query."""
|
|
424
|
+
url = row.get(self.url)
|
|
425
|
+
listed = [row.get(self.events) if self.events else None, *(row.get(field) for field in self.scope.values())]
|
|
426
|
+
readable = all(_items(value) is not None for value in listed)
|
|
427
|
+
return {
|
|
428
|
+
"key": key,
|
|
429
|
+
"url": _public_url(url) if isinstance(url, str) else None,
|
|
430
|
+
"events": _shown(row.get(self.events) if self.events else None),
|
|
431
|
+
"scope": {name: _shown(row.get(field)) for name, field in self.scope.items()},
|
|
432
|
+
"enabled": self._enabled(row) and _receivable(url) and readable,
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
def _wants(self, row: dict, event_type: str) -> bool:
|
|
436
|
+
return _admits(row.get(self.events) if self.events else None, event_type)
|
|
437
|
+
|
|
438
|
+
def _holds(self, row: dict, scope: Mapping[str, str]) -> bool:
|
|
439
|
+
return all(_admits(row.get(self.scope[key]), value) for key, value in scope.items())
|
|
440
|
+
|
|
441
|
+
def _delivery(self, row: dict, key: dict, event: WebhookEvent, payload: bytes, at: int) -> Delivery:
|
|
442
|
+
delivery_id = f"dlv_{uuid.uuid4().hex}"
|
|
443
|
+
secret = row.get(self.secret) if self.secret else None
|
|
444
|
+
unsigned = not (isinstance(secret, str) and secret)
|
|
445
|
+
if unsigned and self.signer.needs_secret and not self.signer.signs_without_secret:
|
|
446
|
+
raise Unsignable(f"no {self.secret} to sign {self.signer.name} with")
|
|
447
|
+
signed = self.signer.headers(delivery_id, event, payload, secret or "", at)
|
|
448
|
+
headers = {"content-type": JSON_TYPE, **signed}
|
|
449
|
+
return Delivery(delivery_id, event, key, row[self.url], headers, payload, at)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
class WebhookOutbox:
|
|
453
|
+
"""The deliveries one world staged, kept only when the call that staged them commits:
|
|
454
|
+
a call that answers an error, breaks the contract or raises keeps none, like its writes.
|
|
455
|
+
Each thread stages its own call's deliveries, so concurrent calls never keep or drop each
|
|
456
|
+
other's. The door that ran the call takes them (:meth:`take`); the last
|
|
457
|
+
:data:`HELD_DELIVERIES` wait for a door that never comes (a world driven in-process)."""
|
|
458
|
+
|
|
459
|
+
def __init__(
|
|
460
|
+
self,
|
|
461
|
+
webhooks: Webhooks | None,
|
|
462
|
+
rows: Callable[[str], list[dict]],
|
|
463
|
+
key: Callable[[str, dict], dict],
|
|
464
|
+
clock: Callable[[], float],
|
|
465
|
+
) -> None:
|
|
466
|
+
self.webhooks = webhooks
|
|
467
|
+
self._rows = rows
|
|
468
|
+
self._key = key
|
|
469
|
+
self._clock = clock
|
|
470
|
+
self._held: list[Delivery] = []
|
|
471
|
+
self._lock = threading.Lock()
|
|
472
|
+
self._calls = threading.local()
|
|
473
|
+
|
|
474
|
+
def _staging(self) -> list[list[Delivery]]:
|
|
475
|
+
if not hasattr(self._calls, "stack"):
|
|
476
|
+
self._calls.stack = []
|
|
477
|
+
return self._calls.stack
|
|
478
|
+
|
|
479
|
+
@contextmanager
|
|
480
|
+
def transaction(self) -> Iterator[None]:
|
|
481
|
+
stack = self._staging()
|
|
482
|
+
stack.append([])
|
|
483
|
+
try:
|
|
484
|
+
yield
|
|
485
|
+
except BaseException:
|
|
486
|
+
stack.pop()
|
|
487
|
+
raise
|
|
488
|
+
self._keep(stack.pop(), stack)
|
|
489
|
+
|
|
490
|
+
def _keep(self, staged: list[Delivery], stack: list[list[Delivery]]) -> None:
|
|
491
|
+
if stack:
|
|
492
|
+
stack[-1].extend(staged)
|
|
493
|
+
return
|
|
494
|
+
with self._lock:
|
|
495
|
+
self._held.extend(staged)
|
|
496
|
+
dropped = len(self._held) - HELD_DELIVERIES
|
|
497
|
+
del self._held[:-HELD_DELIVERIES]
|
|
498
|
+
if dropped > 0:
|
|
499
|
+
log_line(f"webhooks: {dropped} delivery(ies) no door took were let go")
|
|
500
|
+
|
|
501
|
+
def deliver(
|
|
502
|
+
self, event_type: str, body: Any, scope: Mapping[str, Any] | None = None, event_id: str | None = None
|
|
503
|
+
) -> list[str]:
|
|
504
|
+
webhooks = self.webhooks
|
|
505
|
+
if webhooks is None:
|
|
506
|
+
raise WebhookMisuse("deliver needs a [webhooks] table in connector.toml")
|
|
507
|
+
event = webhooks.event(event_type, body, scope, event_id)
|
|
508
|
+
entity = webhooks.subscriptions
|
|
509
|
+
found = webhooks.deliveries(self._rows(entity), lambda row: self._key(entity, row), event, int(self._clock()))
|
|
510
|
+
stack = self._staging()
|
|
511
|
+
staged = sum(len(frame) for frame in stack)
|
|
512
|
+
if staged + len(found) > MAX_DELIVERIES_PER_CALL:
|
|
513
|
+
raise WebhookMisuse(f"a call sends at most {MAX_DELIVERIES_PER_CALL} webhook deliveries")
|
|
514
|
+
self._keep(found, stack)
|
|
515
|
+
return [delivery.id for delivery in found]
|
|
516
|
+
|
|
517
|
+
def clear(self) -> None:
|
|
518
|
+
with self._lock:
|
|
519
|
+
self._held = []
|
|
520
|
+
|
|
521
|
+
def subscriptions(self) -> list[dict]:
|
|
522
|
+
webhooks = self.webhooks
|
|
523
|
+
if webhooks is None:
|
|
524
|
+
return []
|
|
525
|
+
entity = webhooks.subscriptions
|
|
526
|
+
return [webhooks.subscription(row, self._key(entity, row)) for row in self._rows(entity)]
|
|
527
|
+
|
|
528
|
+
def take(self) -> list[Delivery]:
|
|
529
|
+
with self._lock:
|
|
530
|
+
taken, self._held = self._held, []
|
|
531
|
+
return taken
|
|
532
|
+
|
|
533
|
+
@property
|
|
534
|
+
def held(self) -> list[Delivery]:
|
|
535
|
+
with self._lock:
|
|
536
|
+
return list(self._held)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
@dataclass(frozen=True)
|
|
540
|
+
class WebhookAnswer:
|
|
541
|
+
"""What a receiver answered a forwarded delivery: its status, or 0 and why it never answered."""
|
|
542
|
+
|
|
543
|
+
delivery_id: str
|
|
544
|
+
status: int
|
|
545
|
+
duration_ms: float | None
|
|
546
|
+
error: str | None
|
|
547
|
+
|
|
548
|
+
@classmethod
|
|
549
|
+
def parse(cls, value: Any, at: str) -> WebhookAnswer:
|
|
550
|
+
if not isinstance(value, dict) or not isinstance(value.get("id"), str) or not 0 < len(value["id"]) <= 128:
|
|
551
|
+
fail("input", "an answer is {id, status, durationMs?, error?}, its id at most 128 characters", at)
|
|
552
|
+
status = value.get("status")
|
|
553
|
+
if type(status) is not int or not (status == 0 or 100 <= status <= 599):
|
|
554
|
+
fail("input", "status is the receiver's HTTP status, or 0 when it never answered", f"{at}/status")
|
|
555
|
+
duration = value.get("durationMs")
|
|
556
|
+
if duration is not None and (type(duration) not in (int, float) or duration < 0):
|
|
557
|
+
fail("input", "durationMs is a non-negative number", f"{at}/durationMs")
|
|
558
|
+
error = value.get("error")
|
|
559
|
+
if error is not None and not isinstance(error, str):
|
|
560
|
+
fail("input", "error is a string", f"{at}/error")
|
|
561
|
+
return cls(value["id"], status, None if duration is None else float(duration), error and error[:500])
|
|
562
|
+
|
|
563
|
+
def as_json(self) -> dict:
|
|
564
|
+
return {"id": self.delivery_id, "status": self.status, "durationMs": self.duration_ms, "error": self.error}
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
@dataclass(frozen=True)
|
|
568
|
+
class WebhookPoll:
|
|
569
|
+
"""One poll: deliveries after ``after`` in feed ``epoch`` (none: the feed it is on), or none and
|
|
570
|
+
the feed's head when ``tail``; ``answers`` for the deliveries a forwarder sent on; the world's
|
|
571
|
+
subscriptions unless a forwarder, which never shows them, asks for none."""
|
|
572
|
+
|
|
573
|
+
after: int = 0
|
|
574
|
+
limit: int = FEED_PAGE_MAX
|
|
575
|
+
answers: tuple[WebhookAnswer, ...] = ()
|
|
576
|
+
epoch: str | None = None
|
|
577
|
+
tail: bool = False
|
|
578
|
+
subscriptions: bool = True
|
|
579
|
+
targets: bool = False
|
|
580
|
+
|
|
581
|
+
@classmethod
|
|
582
|
+
def parse(cls, body: dict) -> WebhookPoll:
|
|
583
|
+
after, limit, answers = body.get("after", 0), body.get("limit", FEED_PAGE_MAX), body.get("answers", [])
|
|
584
|
+
epoch, tail, listing = body.get("epoch"), body.get("tail", False), body.get("subscriptions", True)
|
|
585
|
+
targets = body.get("targets", False)
|
|
586
|
+
if type(after) is not int or after < 0:
|
|
587
|
+
fail("input", "after must be the last delivery seq read, 0 before the first", "/after")
|
|
588
|
+
if type(limit) is not int or not 1 <= limit <= FEED_PAGE_MAX:
|
|
589
|
+
fail("input", f"limit must be an integer from 1 to {FEED_PAGE_MAX}", "/limit")
|
|
590
|
+
if not isinstance(answers, list) or len(answers) > FEED_PAGE_MAX:
|
|
591
|
+
fail("input", f"answers must be a list of at most {FEED_PAGE_MAX}", "/answers")
|
|
592
|
+
if epoch is not None and not (isinstance(epoch, str) and FEED_EPOCH.fullmatch(epoch)):
|
|
593
|
+
fail("input", "epoch must be the epoch a page answered", "/epoch")
|
|
594
|
+
if type(tail) is not bool:
|
|
595
|
+
fail("input", "tail is a boolean", "/tail")
|
|
596
|
+
if type(listing) is not bool:
|
|
597
|
+
fail("input", "subscriptions is a boolean", "/subscriptions")
|
|
598
|
+
if type(targets) is not bool:
|
|
599
|
+
fail("input", "targets is a boolean", "/targets")
|
|
600
|
+
parsed = tuple(WebhookAnswer.parse(answer, f"/answers/{index}") for index, answer in enumerate(answers))
|
|
601
|
+
return cls(after, limit, parsed, epoch, tail, listing, targets)
|
|
602
|
+
|
|
603
|
+
def as_json(self) -> dict:
|
|
604
|
+
answers = [answer.as_json() for answer in self.answers]
|
|
605
|
+
shape = {"after": self.after, "limit": self.limit, "epoch": self.epoch, "tail": self.tail}
|
|
606
|
+
return {**shape, "answers": answers, "subscriptions": self.subscriptions, "targets": self.targets}
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
class WebhookFeed:
|
|
610
|
+
"""The deliveries one session's world sent, numbered in order, the last :data:`FEED_DELIVERIES`
|
|
611
|
+
(at most :data:`FEED_BYTES`) kept whole, signed headers and exact payload, for a forwarder to
|
|
612
|
+
replay. In memory: a session that starts again on another host starts a fresh feed under a new
|
|
613
|
+
``epoch``, and a poll naming the old one (or a seq the feed never reached) reads it from the
|
|
614
|
+
start with ``reset``. Each delivery takes one receiver answer; the ledger keeps every delivery."""
|
|
615
|
+
|
|
616
|
+
def __init__(self, size: int = FEED_DELIVERIES, max_bytes: int = FEED_BYTES) -> None:
|
|
617
|
+
self.epoch = uuid.uuid4().hex
|
|
618
|
+
self._size = size
|
|
619
|
+
self._max_bytes = max_bytes
|
|
620
|
+
self._held: list[tuple[int, Delivery]] = []
|
|
621
|
+
self._bytes = 0
|
|
622
|
+
self._seq = 0
|
|
623
|
+
self._answered: set[str] = set()
|
|
624
|
+
self._sent: dict[str, int] = {}
|
|
625
|
+
self._lock = threading.Lock()
|
|
626
|
+
|
|
627
|
+
def append(self, deliveries: Iterable[Delivery]) -> None:
|
|
628
|
+
with self._lock:
|
|
629
|
+
for delivery in deliveries:
|
|
630
|
+
self._seq += 1
|
|
631
|
+
self._held.append((self._seq, delivery))
|
|
632
|
+
self._bytes += delivery.size
|
|
633
|
+
key = _subscription_key(delivery.subscription)
|
|
634
|
+
self._sent[key] = self._sent.get(key, 0) + 1
|
|
635
|
+
self._trim()
|
|
636
|
+
|
|
637
|
+
def _trim(self) -> None:
|
|
638
|
+
while self._held and (len(self._held) > self._size or self._bytes > self._max_bytes):
|
|
639
|
+
_, gone = self._held.pop(0)
|
|
640
|
+
self._bytes -= gone.size
|
|
641
|
+
self._answered.discard(gone.id)
|
|
642
|
+
|
|
643
|
+
def after(
|
|
644
|
+
self, seq: int, limit: int = FEED_PAGE_MAX, epoch: str | None = None, tail: bool = False, targets: bool = False
|
|
645
|
+
) -> dict:
|
|
646
|
+
with self._lock:
|
|
647
|
+
reset = not tail and ((epoch is not None and epoch != self.epoch) or seq > self._seq)
|
|
648
|
+
seq = self._seq if tail else 0 if reset else seq
|
|
649
|
+
newer = [(n, d) for n, d in self._held if n > seq]
|
|
650
|
+
first = self._held[0][0] if self._held else self._seq + 1
|
|
651
|
+
page = _bounded_page(newer, limit)
|
|
652
|
+
return {
|
|
653
|
+
"epoch": self.epoch,
|
|
654
|
+
"reset": reset,
|
|
655
|
+
"deliveries": [{"seq": n, **d.as_json(targets)} for n, d in page],
|
|
656
|
+
"next": page[-1][0] if page else seq,
|
|
657
|
+
"more": len(newer) > len(page),
|
|
658
|
+
"missed": max(0, first - seq - 1),
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
def answer(self, delivery_id: str) -> Delivery | None:
|
|
662
|
+
"""The delivery an answer is for, the first time only; None for one already answered or let go."""
|
|
663
|
+
with self._lock:
|
|
664
|
+
delivery = next((d for _, d in self._held if d.id == delivery_id), None)
|
|
665
|
+
if delivery is None or delivery_id in self._answered:
|
|
666
|
+
return None
|
|
667
|
+
self._answered.add(delivery_id)
|
|
668
|
+
return delivery
|
|
669
|
+
|
|
670
|
+
def sent_to(self, key: dict) -> int:
|
|
671
|
+
with self._lock:
|
|
672
|
+
return self._sent.get(_subscription_key(key), 0)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _subscription_key(key: dict) -> str:
|
|
676
|
+
return json.dumps(key, sort_keys=True, default=str)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _bounded_page(newer: list[tuple[int, Delivery]], limit: int) -> list[tuple[int, Delivery]]:
|
|
680
|
+
page: list[tuple[int, Delivery]] = []
|
|
681
|
+
spent = 0
|
|
682
|
+
for seq, delivery in newer[:limit]:
|
|
683
|
+
if page and spent + delivery.size > PAGE_BYTES:
|
|
684
|
+
break
|
|
685
|
+
page.append((seq, delivery))
|
|
686
|
+
spent += delivery.size
|
|
687
|
+
return page
|