@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -16,6 +16,11 @@ or a field named only for what it holds (:data:`NAMED_BY_HOLDER`: ``value``,
|
|
|
16
16
|
``pan``), which elsewhere name other things (``boarding_pass``);
|
|
17
17
|
* it is a ``number``, ``no`` or ``pan`` under a ``card``.
|
|
18
18
|
|
|
19
|
+
Every string kept is scrubbed in place (:meth:`CallRedaction.scrub`), the rest
|
|
20
|
+
of it kept: the session's credential as sent and percent-encoded, the value of a
|
|
21
|
+
query parameter the world's ``[auth]`` carries a secret in (a query key, a signed
|
|
22
|
+
link's ``signature``) wherever a link holding it sits, and a JSON Web Token.
|
|
23
|
+
|
|
19
24
|
A value is a credential by its shape, whatever its field: a provider key, an
|
|
20
25
|
AWS key id or a bearer header (:data:`~gatewaysdk.environment.secret_values.SECRET_VALUE`,
|
|
21
26
|
the browser recorder's rule), the further token shapes of
|
|
@@ -30,8 +35,9 @@ Each string or number so named or shaped reads :data:`REDACTED`; the key stays,
|
|
|
30
35
|
so a grader can still ask whether it was sent. Booleans, nulls and every other
|
|
31
36
|
field are kept as sent, so a grader reads what the agent asked, with what
|
|
32
37
|
no reply can carry read as text: an unpaired surrogate as U+FFFD, a NaN or an
|
|
33
|
-
infinity as ``"NaN"``, ``"Infinity"``, ``"-Infinity"``. A query
|
|
34
|
-
|
|
38
|
+
infinity as ``"NaN"``, ``"Infinity"``, ``"-Infinity"``. A query parameter the
|
|
39
|
+
world's ``[auth]`` carries a secret in (a query-auth key, a signed link's
|
|
40
|
+
signature) is dropped. A request holding more than
|
|
35
41
|
:data:`MAX_LOGGED_VALUES` values, keys and key parts, or :data:`MAX_LOGGED_CHARS`
|
|
36
42
|
characters of text, its keys included, is too large to keep
|
|
37
43
|
(:class:`RequestTooLarge`): its whole request shares one :class:`RequestBudget`.
|
|
@@ -43,15 +49,18 @@ import re
|
|
|
43
49
|
import urllib.parse
|
|
44
50
|
from dataclasses import dataclass, replace
|
|
45
51
|
from functools import cached_property
|
|
46
|
-
from typing import Any, Iterator
|
|
52
|
+
from typing import Any, Callable, Iterable, Iterator
|
|
47
53
|
|
|
48
54
|
from ..secret_values import SECRET_TOKEN, SECRET_VALUE, basic_credentials
|
|
49
55
|
from .connector import PLACEHOLDER, Operation, Spec
|
|
56
|
+
from .door import open_label
|
|
50
57
|
from .handler_errors import carried_scalar
|
|
51
58
|
|
|
52
59
|
REDACTED = "[redacted]"
|
|
53
60
|
MAX_LOGGED_VALUES = 100_000
|
|
54
61
|
MAX_LOGGED_CHARS = 1 << 20
|
|
62
|
+
SCRUB_MARGIN = 8 * 1024
|
|
63
|
+
MIN_SCRUBBED_SECRET = 16
|
|
55
64
|
|
|
56
65
|
CREDENTIAL_WORDS = frozenset(
|
|
57
66
|
{
|
|
@@ -161,7 +170,37 @@ KEPT_SUFFIXES = (
|
|
|
161
170
|
"deltatoken",
|
|
162
171
|
"preauthorization",
|
|
163
172
|
)
|
|
173
|
+
KEPT_HEADERS = frozenset(
|
|
174
|
+
{
|
|
175
|
+
"accept",
|
|
176
|
+
"cache-control",
|
|
177
|
+
"content-encoding",
|
|
178
|
+
"content-length",
|
|
179
|
+
"content-type",
|
|
180
|
+
"host",
|
|
181
|
+
"idempotency-key",
|
|
182
|
+
"linear-delivery",
|
|
183
|
+
"linear-event",
|
|
184
|
+
"origin",
|
|
185
|
+
"svix-id",
|
|
186
|
+
"svix-timestamp",
|
|
187
|
+
"user-agent",
|
|
188
|
+
"x-github-delivery",
|
|
189
|
+
"x-github-event",
|
|
190
|
+
"x-forwarded-proto",
|
|
191
|
+
"x-request-id",
|
|
192
|
+
}
|
|
193
|
+
)
|
|
194
|
+
KEPT_HEADER_PREFIXES = ("accept-", "if-", "sec-fetch-")
|
|
195
|
+
HEADER_NAME_CHARS = 256
|
|
164
196
|
_WORD = re.compile(r"[A-Z]?[a-z]+\d*|[A-Z]+\d*(?![a-z])|\d+")
|
|
197
|
+
_QUERY_LEAD = r"(?:[?&;]|%(?:25)?26|%(?:25)?3[Ff]|\\u0026|\\u003[Ff])"
|
|
198
|
+
_QUERY_EQUALS = r"(?:=|%(?:25)?3[Dd]|\\u003[Dd])"
|
|
199
|
+
_QUERY_VALUE = r"[^&#\"'\s\\<>,;)]*"
|
|
200
|
+
_EMBEDDED_TOKEN = re.compile(
|
|
201
|
+
r"(?:(?<![-_A-Za-z0-9])|(?<=%[0-9A-Fa-f]{2})|(?<=\\u00[0-9A-Fa-f]{2}))"
|
|
202
|
+
r"eyJ[-_A-Za-z0-9]{8,}\.eyJ[-_A-Za-z0-9]{8,}\.[-_A-Za-z0-9]+"
|
|
203
|
+
)
|
|
165
204
|
_CARD_DIGITS = re.compile(r"^\d(?:[ -]?\d){12,18}$")
|
|
166
205
|
|
|
167
206
|
|
|
@@ -194,6 +233,11 @@ def _card_digits(text: str) -> bool:
|
|
|
194
233
|
return bool(_CARD_DIGITS.match(text)) and _luhn(re.sub(r"[ -]", "", text))
|
|
195
234
|
|
|
196
235
|
|
|
236
|
+
def _secret_segment(segment: str, secret: str | None) -> bool:
|
|
237
|
+
decoded = urllib.parse.unquote(segment)
|
|
238
|
+
return _shaped_like_a_secret(decoded) or (bool(secret) and secret in decoded)
|
|
239
|
+
|
|
240
|
+
|
|
197
241
|
def _shaped_like_a_secret(value: Any) -> bool:
|
|
198
242
|
if not isinstance(value, str):
|
|
199
243
|
return False
|
|
@@ -269,7 +313,10 @@ class RequestBudget:
|
|
|
269
313
|
keys and key parts, and :data:`MAX_LOGGED_CHARS` characters of text, its
|
|
270
314
|
keys included, an integer counted by its bytes. Each is charged before it is read, so a request past it costs
|
|
271
315
|
no more than the budget; what it spent is what the logged entry holds, which
|
|
272
|
-
the call log charges to its own, per-session budget.
|
|
316
|
+
the call log charges to its own, per-session budget. It refuses rather than
|
|
317
|
+
fills, so it is never :attr:`full`."""
|
|
318
|
+
|
|
319
|
+
full = False
|
|
273
320
|
|
|
274
321
|
def __init__(self) -> None:
|
|
275
322
|
self.values = MAX_LOGGED_VALUES
|
|
@@ -284,6 +331,43 @@ class RequestBudget:
|
|
|
284
331
|
if self.values < 0 or self.chars < 0:
|
|
285
332
|
raise RequestTooLarge
|
|
286
333
|
|
|
334
|
+
def text(self, value: str) -> str:
|
|
335
|
+
return value
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
class PrefixBudget(RequestBudget):
|
|
339
|
+
"""A walk that keeps a value's leading part in the order JSON writes it: it
|
|
340
|
+
never refuses, and once ``size`` values or characters are spent it is
|
|
341
|
+
:attr:`full`, so the walk adds nothing more and records in :attr:`cut` that
|
|
342
|
+
something was left. What it walked writes at least ``size`` characters of
|
|
343
|
+
JSON, so the first ``size`` bytes of the part it kept are the first ``size``
|
|
344
|
+
bytes of the whole."""
|
|
345
|
+
|
|
346
|
+
def __init__(self, size: int) -> None:
|
|
347
|
+
self.size = size
|
|
348
|
+
self.values = size
|
|
349
|
+
self.chars = size
|
|
350
|
+
self.cut = False
|
|
351
|
+
|
|
352
|
+
@property
|
|
353
|
+
def full(self) -> bool:
|
|
354
|
+
return self.values <= 0 or self.chars <= 0
|
|
355
|
+
|
|
356
|
+
def spent(self) -> tuple[int, int]:
|
|
357
|
+
return self.size - self.values, self.size - self.chars
|
|
358
|
+
|
|
359
|
+
def spend(self, values: int, chars: int) -> None:
|
|
360
|
+
self.values -= values
|
|
361
|
+
self.chars -= chars
|
|
362
|
+
|
|
363
|
+
def text(self, value: str) -> str:
|
|
364
|
+
"""``value`` to its first ``size`` characters and :data:`SCRUB_MARGIN` more: no
|
|
365
|
+
more of one string can be kept, and the margin holds whole any secret the
|
|
366
|
+
kept part ends inside, so a scrub finds it."""
|
|
367
|
+
kept = value[: self.size + SCRUB_MARGIN]
|
|
368
|
+
self.cut = self.cut or len(kept) < len(value)
|
|
369
|
+
return kept
|
|
370
|
+
|
|
287
371
|
|
|
288
372
|
def _size(value: Any) -> int:
|
|
289
373
|
"""What a scalar holds beyond one value: a string's characters, an integer's bytes."""
|
|
@@ -294,42 +378,146 @@ def _size(value: Any) -> int:
|
|
|
294
378
|
return 0
|
|
295
379
|
|
|
296
380
|
|
|
297
|
-
def _kept(value: Any, name: FieldName, budget: RequestBudget) -> Any:
|
|
381
|
+
def _kept(value: Any, name: FieldName, budget: RequestBudget, scrub: Callable[[str], str]) -> Any:
|
|
298
382
|
budget.spend(1, _size(value))
|
|
299
383
|
if isinstance(value, dict):
|
|
300
|
-
return {carried_scalar(key): _kept(item, name.child(key), budget) for key, item in _keys(value, budget)}
|
|
384
|
+
return {carried_scalar(key): _kept(item, name.child(key), budget, scrub) for key, item in _keys(value, budget)}
|
|
301
385
|
if isinstance(value, list):
|
|
302
386
|
item_name = name.child("*")
|
|
303
|
-
return [_kept(item, item_name, budget) for item in value]
|
|
387
|
+
return [_kept(item, item_name, budget, scrub) for item in _items(value, budget)]
|
|
304
388
|
if _scalar(value) and name.redacts(value):
|
|
305
389
|
return REDACTED
|
|
306
|
-
return carried_scalar(value)
|
|
390
|
+
return carried_scalar(scrub(budget.text(value)) if isinstance(value, str) else value)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _spellings(secret: str | None) -> list[str]:
|
|
394
|
+
"""``secret`` as it may sit in a body: as sent, and percent-encoded the two ways a URL
|
|
395
|
+
carries it, and the capability label an open session's hostname derives from it; longest
|
|
396
|
+
first, so one spelling never leaves part of another. A secret shorter than
|
|
397
|
+
:data:`MIN_SCRUBBED_SECRET` (a local pin such as ``conform``; a World Host token has 32
|
|
398
|
+
characters) is ordinary text and has none."""
|
|
399
|
+
if not secret or len(secret) < MIN_SCRUBBED_SECRET:
|
|
400
|
+
return []
|
|
401
|
+
forms = {secret, urllib.parse.quote(secret, safe=""), urllib.parse.quote_plus(secret), open_label(secret)}
|
|
402
|
+
return sorted(forms, key=len, reverse=True)
|
|
307
403
|
|
|
308
404
|
|
|
309
405
|
def _keys(value: dict, budget: RequestBudget) -> Iterator[tuple[str, Any]]:
|
|
310
|
-
for key, item in value.items():
|
|
406
|
+
for key, item in _items(value.items(), budget):
|
|
311
407
|
text = str(key)
|
|
312
408
|
budget.spend(text.count(".") + 1, len(text))
|
|
313
409
|
yield text, item
|
|
314
410
|
|
|
315
411
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
412
|
+
def _items(items: Iterable[Any], budget: RequestBudget) -> Iterator[Any]:
|
|
413
|
+
for item in items:
|
|
414
|
+
if budget.full:
|
|
415
|
+
budget.cut = True
|
|
416
|
+
return
|
|
417
|
+
yield item
|
|
418
|
+
|
|
319
419
|
|
|
320
|
-
|
|
321
|
-
|
|
420
|
+
class CallRedaction:
|
|
421
|
+
"""The call log's redaction (module docstring), for one session of one world:
|
|
422
|
+
``credential_params`` are the query parameters its ``[auth]`` carries a secret in
|
|
423
|
+
(a query-auth world's key, a signed link's signature), dropped from every logged
|
|
424
|
+
query; ``credential_header`` the header its scheme carries a key or a minted token
|
|
425
|
+
in, whatever it is named, kept as :data:`REDACTED` wherever headers are kept;
|
|
426
|
+
``secret`` the session's credential, which no string kept holds."""
|
|
427
|
+
|
|
428
|
+
def __init__(
|
|
429
|
+
self,
|
|
430
|
+
credential_params: frozenset[str] = frozenset(),
|
|
431
|
+
credential_header: str | None = None,
|
|
432
|
+
secret: str | None = None,
|
|
433
|
+
):
|
|
434
|
+
self.credential_params = credential_params
|
|
435
|
+
self.credential_header = (credential_header or "").lower()
|
|
436
|
+
self.secret = secret
|
|
437
|
+
self._spellings = _spellings(secret)
|
|
438
|
+
self._names = tuple(sorted(credential_params))
|
|
439
|
+
alternatives = "|".join(re.escape(name) for name in self._names)
|
|
440
|
+
lead = f"({_QUERY_LEAD}(?:{alternatives}){_QUERY_EQUALS}){_QUERY_VALUE}"
|
|
441
|
+
self._carried = re.compile(lead) if alternatives else None
|
|
322
442
|
|
|
323
443
|
@classmethod
|
|
324
|
-
def of(cls, spec: Spec) -> CallRedaction:
|
|
325
|
-
|
|
444
|
+
def of(cls, spec: Spec | None, secret: str | None = None) -> CallRedaction:
|
|
445
|
+
if spec is None:
|
|
446
|
+
return cls(secret=secret)
|
|
447
|
+
return cls(spec.auth.secret_params, spec.auth.scheme.secret_header, secret)
|
|
448
|
+
|
|
449
|
+
def value(self, value: Any, budget: RequestBudget | None = None) -> Any:
|
|
450
|
+
return _kept(value, ROOT, budget or RequestBudget(), self.scrub)
|
|
451
|
+
|
|
452
|
+
def prefix(self, value: Any, size: int) -> tuple[Any, bool]:
|
|
453
|
+
"""``value`` redacted as far as its first ``size`` values or characters
|
|
454
|
+
(:class:`PrefixBudget`), and whether any of it was left out."""
|
|
455
|
+
budget = PrefixBudget(size)
|
|
456
|
+
return _kept(value, ROOT, budget, self.scrub), budget.cut
|
|
457
|
+
|
|
458
|
+
def headers(self, headers: dict[str, str], secret: str | None, size: int) -> dict[str, str]:
|
|
459
|
+
"""A request's headers as the request ledger keeps them, the first ``size``
|
|
460
|
+
characters of them, every name kept to its first :data:`HEADER_NAME_CHARS`. A
|
|
461
|
+
value is kept only under a name that says what it is (:data:`KEPT_HEADERS`,
|
|
462
|
+
:data:`KEPT_HEADER_PREFIXES`) and then still redacted by its shape; every other
|
|
463
|
+
value, however a vendor names its key, the world's own credential header and any
|
|
464
|
+
value carrying ``secret`` (the session's token) read :data:`REDACTED`."""
|
|
465
|
+
kept, _ = self.prefix(dict(headers), size)
|
|
466
|
+
return {
|
|
467
|
+
name[:HEADER_NAME_CHARS]: self._header(name, headers.get(name, ""), value, secret)
|
|
468
|
+
for name, value in kept.items()
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
def _header(self, name: str, sent: str, kept: Any, secret: str | None) -> Any:
|
|
472
|
+
lowered = name.lower()
|
|
473
|
+
described = lowered in KEPT_HEADERS or lowered.startswith(KEPT_HEADER_PREFIXES)
|
|
474
|
+
carried = bool(secret) and secret in sent
|
|
475
|
+
return kept if described and lowered != self.credential_header and not carried else REDACTED
|
|
476
|
+
|
|
477
|
+
def scrub(self, text: str) -> str:
|
|
478
|
+
"""``text`` with every secret it embeds read :data:`REDACTED` in place: the
|
|
479
|
+
session's credential as sent and percent-encoded, an open session's capability
|
|
480
|
+
label (its hostname as a request or a link names it), the value of each query
|
|
481
|
+
parameter the world's ``[auth]`` carries a secret in (in a link a world echoes
|
|
482
|
+
or an agent passes on) and a JSON Web Token. Each is found in time linear in
|
|
483
|
+
``text``."""
|
|
484
|
+
for spelling in self._spellings:
|
|
485
|
+
text = text.replace(spelling, REDACTED)
|
|
486
|
+
if self._carried is not None and any(name in text for name in self._names):
|
|
487
|
+
text = self._carried.sub(rf"\1{REDACTED}", text)
|
|
488
|
+
return _EMBEDDED_TOKEN.sub(REDACTED, text) if "eyJ" in text else text
|
|
326
489
|
|
|
327
490
|
@staticmethod
|
|
328
|
-
def
|
|
329
|
-
|
|
491
|
+
def path_without_secret(path: str, secret: str | None) -> str:
|
|
492
|
+
"""``path`` with each segment carrying ``secret`` (the session's token) read :data:`REDACTED`."""
|
|
493
|
+
if not secret:
|
|
494
|
+
return path
|
|
495
|
+
return "/".join(REDACTED if secret in urllib.parse.unquote(part) else part for part in path.split("/"))
|
|
496
|
+
|
|
497
|
+
@staticmethod
|
|
498
|
+
def without_secret(value: Any, secret: str | None) -> Any:
|
|
499
|
+
"""``value`` with every string carrying ``secret`` (the session's token), at any
|
|
500
|
+
depth, read :data:`REDACTED`."""
|
|
501
|
+
if not secret:
|
|
502
|
+
return value
|
|
503
|
+
if isinstance(value, str):
|
|
504
|
+
return REDACTED if secret in value else value
|
|
505
|
+
if isinstance(value, dict):
|
|
506
|
+
return {key: CallRedaction.without_secret(item, secret) for key, item in value.items()}
|
|
507
|
+
if isinstance(value, list):
|
|
508
|
+
return [CallRedaction.without_secret(item, secret) for item in value]
|
|
509
|
+
return value
|
|
510
|
+
|
|
511
|
+
def raw_path(self, path: str, secret: str | None) -> str:
|
|
512
|
+
"""A path no route matched, so no argument is named: each segment shaped like a
|
|
513
|
+
secret, or carrying ``secret`` (the session's token), reads :data:`REDACTED`,
|
|
514
|
+
and every other one is scrubbed (:meth:`scrub`)."""
|
|
515
|
+
return "/".join(
|
|
516
|
+
REDACTED if _secret_segment(segment, secret) else self.scrub(segment) for segment in path.split("/")
|
|
517
|
+
)
|
|
330
518
|
|
|
331
519
|
def query(self, query_all: dict, budget: RequestBudget | None = None) -> dict:
|
|
332
|
-
kept = {k: (v[0] if len(v) == 1 else list(v)) for k, v in query_all.items() if k
|
|
520
|
+
kept = {k: (v[0] if len(v) == 1 else list(v)) for k, v in query_all.items() if k not in self.credential_params}
|
|
333
521
|
return self.value(kept, budget)
|
|
334
522
|
|
|
335
523
|
def path(self, operation: Operation, path: str, args: dict) -> str:
|
|
@@ -4,11 +4,11 @@ The TypeScript twin is ``worldData/cast.ts``; ``tests/conformance/csv_cast.json`
|
|
|
4
4
|
them to the same answers. ``integer`` and ``number`` parse as numbers (a number is
|
|
5
5
|
always a float), ``boolean`` takes true/false/1/0 in any case, ``array`` and
|
|
6
6
|
``object`` parse as JSON of that shape, ``string`` is the text as given. An empty
|
|
7
|
-
cell is ``None`` when the field's type admits null,
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
cell is ``None`` when the field's type admits null, the empty string for a required
|
|
8
|
+
string, and left out otherwise. A union type (``integer | string``) tries each member
|
|
9
|
+
in order, ``string`` last. A cell that fits none stays its text, as does a column the
|
|
10
|
+
schema does not name: the contract's check then refuses the row, naming its entity,
|
|
11
|
+
key and field, as it refuses a JSON row, so a file reports every bad row at once.
|
|
12
12
|
|
|
13
13
|
Standard library only, so the bridge and the api bundle can load it.
|
|
14
14
|
"""
|
|
@@ -23,8 +23,6 @@ from dataclasses import dataclass
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Any
|
|
25
25
|
|
|
26
|
-
from .validation import fail
|
|
27
|
-
|
|
28
26
|
INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
|
|
29
27
|
NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
|
|
30
28
|
BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
|
|
@@ -51,21 +49,6 @@ class FieldType:
|
|
|
51
49
|
return described
|
|
52
50
|
|
|
53
51
|
|
|
54
|
-
def _article(kind: str) -> str:
|
|
55
|
-
return "an" if kind[:1].lower() in "aeiou" else "a"
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def refuse(where: str, field: str, cell: str | None, kind: str) -> None:
|
|
59
|
-
"""The row-level refusal a cell that will not cast earns (same words as the TypeScript twin)."""
|
|
60
|
-
if cell is None:
|
|
61
|
-
fail("rows", f"{where}: {field} is empty; the field is a required {kind}", where)
|
|
62
|
-
fail(
|
|
63
|
-
"rows",
|
|
64
|
-
f"{where}: {field} = {json.dumps(cell)} is not {_article(kind)} {kind}",
|
|
65
|
-
where,
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
|
|
69
52
|
def type_members(kind: str | None) -> list[str]:
|
|
70
53
|
"""The typed members of a field's ``type`` (``"integer | null"`` -> integer, null); an
|
|
71
54
|
unknown word (a ``$ref`` name) is dropped."""
|
|
@@ -123,10 +106,11 @@ class FieldCaster:
|
|
|
123
106
|
self.required = required
|
|
124
107
|
self.members = type_members(kind)
|
|
125
108
|
|
|
126
|
-
def cast(self, cell: str | None
|
|
127
|
-
"""The typed value; :data:`OMIT` to leave the field out of the row
|
|
109
|
+
def cast(self, cell: str | None) -> Any:
|
|
110
|
+
"""The typed value; :data:`OMIT` to leave the field out of the row; the cell as given when
|
|
111
|
+
it fits no member, for the contract's check to refuse."""
|
|
128
112
|
if cell is None:
|
|
129
|
-
return self._empty(
|
|
113
|
+
return self._empty()
|
|
130
114
|
if not self.members:
|
|
131
115
|
return cell
|
|
132
116
|
text = cell.strip()
|
|
@@ -134,18 +118,14 @@ class FieldCaster:
|
|
|
134
118
|
value = _CAST.get(member, _no_cast)(text)
|
|
135
119
|
if value is not _MISSING:
|
|
136
120
|
return value
|
|
137
|
-
|
|
138
|
-
return cell
|
|
139
|
-
refuse(where, self.name, cell, self.type or "value")
|
|
121
|
+
return cell
|
|
140
122
|
|
|
141
|
-
def _empty(self
|
|
123
|
+
def _empty(self) -> Any:
|
|
142
124
|
if not self.members or "null" in self.members:
|
|
143
125
|
return None
|
|
144
|
-
if
|
|
145
|
-
return OMIT
|
|
146
|
-
if "string" in self.members:
|
|
126
|
+
if self.required and "string" in self.members:
|
|
147
127
|
return ""
|
|
148
|
-
|
|
128
|
+
return OMIT
|
|
149
129
|
|
|
150
130
|
|
|
151
131
|
#: What :meth:`FieldCaster.cast` answers for an empty optional cell: the field stays out of the row.
|
|
@@ -158,11 +138,11 @@ class RowCaster:
|
|
|
158
138
|
def __init__(self, fields: Iterable[FieldType]):
|
|
159
139
|
self._fields = {f.name: FieldCaster(f.name, f.type, f.required) for f in fields}
|
|
160
140
|
|
|
161
|
-
def row(self, record: dict[str, str | None]
|
|
141
|
+
def row(self, record: dict[str, str | None]) -> dict:
|
|
162
142
|
row: dict = {}
|
|
163
143
|
for name, cell in record.items():
|
|
164
144
|
caster = self._fields.get(name)
|
|
165
|
-
value = cell if caster is None else caster.cast(cell
|
|
145
|
+
value = cell if caster is None else caster.cast(cell)
|
|
166
146
|
if value is not OMIT:
|
|
167
147
|
row[name] = value
|
|
168
148
|
return row
|
|
@@ -899,7 +899,10 @@ def bind_connection(bundle: Path, contract: Contract) -> None:
|
|
|
899
899
|
return
|
|
900
900
|
from .connector import parse_spec
|
|
901
901
|
|
|
902
|
-
parse_spec(spec_path.read_text(encoding="utf-8"))
|
|
902
|
+
spec = parse_spec(spec_path.read_text(encoding="utf-8"))
|
|
903
|
+
spec.check_entities(contract.entities)
|
|
904
|
+
if spec.webhooks is not None:
|
|
905
|
+
spec.webhooks.check_contract(contract.entities)
|
|
903
906
|
|
|
904
907
|
|
|
905
908
|
def check_bundle(bundle: str | Path) -> Contract:
|
|
@@ -26,7 +26,6 @@ body the document's example does not supply.
|
|
|
26
26
|
|
|
27
27
|
from __future__ import annotations
|
|
28
28
|
|
|
29
|
-
import base64
|
|
30
29
|
import json
|
|
31
30
|
import threading
|
|
32
31
|
import urllib.error
|
|
@@ -34,16 +33,20 @@ import urllib.parse
|
|
|
34
33
|
import urllib.request
|
|
35
34
|
from dataclasses import dataclass, field
|
|
36
35
|
from pathlib import Path
|
|
37
|
-
from typing import Any
|
|
36
|
+
from typing import Any, Literal
|
|
38
37
|
|
|
39
38
|
from .._toml import tomllib
|
|
40
39
|
from .api import WorldState, load_api_settings, serve
|
|
41
40
|
from .connector import SPEC, Operation, load_spec
|
|
41
|
+
from .door import Door
|
|
42
42
|
from .validation import fail
|
|
43
43
|
|
|
44
44
|
PIN = "conform"
|
|
45
45
|
|
|
46
46
|
|
|
47
|
+
BODILESS_STATUSES = (204, 205)
|
|
48
|
+
|
|
49
|
+
|
|
47
50
|
def load_openapi(path: str | Path) -> dict:
|
|
48
51
|
return parse_openapi(Path(path).read_text(encoding="utf-8"), str(path))
|
|
49
52
|
|
|
@@ -306,7 +309,7 @@ class Outcome:
|
|
|
306
309
|
documented: bool
|
|
307
310
|
status: int | None = None
|
|
308
311
|
expected: list[str] = field(default_factory=list)
|
|
309
|
-
schema:
|
|
312
|
+
schema: Literal["ok", "violations", "no-schema", "unchecked"] = "unchecked"
|
|
310
313
|
problems: list[str] = field(default_factory=list)
|
|
311
314
|
request: dict = field(default_factory=dict)
|
|
312
315
|
|
|
@@ -316,7 +319,7 @@ class Outcome:
|
|
|
316
319
|
self.documented
|
|
317
320
|
and self.status is not None
|
|
318
321
|
and str(self.status) in self.expected
|
|
319
|
-
and self.schema in ("ok", "no-schema")
|
|
322
|
+
and self.schema in ("ok", "no-schema", "no-body")
|
|
320
323
|
)
|
|
321
324
|
|
|
322
325
|
|
|
@@ -341,27 +344,6 @@ class Report:
|
|
|
341
344
|
return {"world": self.world, "summary": self.summary, "operations": [o.__dict__ for o in self.outcomes]}
|
|
342
345
|
|
|
343
346
|
|
|
344
|
-
def _credentials(base: str, spec, settings) -> dict[str, str]:
|
|
345
|
-
"""Headers that get the conformance run through the world's own auth, using the pin."""
|
|
346
|
-
if spec.scheme == "none":
|
|
347
|
-
return {}
|
|
348
|
-
if spec.scheme == "header":
|
|
349
|
-
return {spec.auth_name or "": PIN}
|
|
350
|
-
if spec.scheme == "bearer":
|
|
351
|
-
return {"Authorization": f"Bearer {PIN}"}
|
|
352
|
-
if spec.scheme == "basic":
|
|
353
|
-
return {"Authorization": "Basic " + base64.b64encode(PIN.encode()).decode()}
|
|
354
|
-
if spec.scheme == "query":
|
|
355
|
-
return {}
|
|
356
|
-
form = urllib.parse.urlencode(
|
|
357
|
-
{"grant_type": "client_credentials", "client_id": "conform", "client_secret": PIN}
|
|
358
|
-
).encode()
|
|
359
|
-
prefix = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
|
|
360
|
-
request = urllib.request.Request(base + prefix + (settings.token_path or "/oauth/token"), data=form, method="POST")
|
|
361
|
-
with urllib.request.urlopen(request, timeout=10) as reply:
|
|
362
|
-
return {"Authorization": "Bearer " + json.loads(reply.read())["access_token"]}
|
|
363
|
-
|
|
364
|
-
|
|
365
347
|
def request_samples(world: str | Path) -> dict[str, tuple[dict[str, str], Any]]:
|
|
366
348
|
"""Per operation, the ``[operations.api] sample_headers`` (header -> value) and
|
|
367
349
|
``sample_body`` (a table or array, sent as JSON) declared in ``connector.toml``.
|
|
@@ -420,13 +402,28 @@ def _first_value(rows: list[dict], path: str) -> Any:
|
|
|
420
402
|
return None
|
|
421
403
|
|
|
422
404
|
|
|
405
|
+
def _keyed_value(state: WorldState, name: str) -> Any:
|
|
406
|
+
for entity, declaration in state.contract.entities.items():
|
|
407
|
+
if name in declaration["primary_key"]:
|
|
408
|
+
value = _first_value(state.rows(entity), name)
|
|
409
|
+
if value is not None:
|
|
410
|
+
return value
|
|
411
|
+
return None
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _own_key_value(state: WorldState, entity: str) -> Any:
|
|
415
|
+
key = state.contract.entities[entity]["primary_key"]
|
|
416
|
+
return _first_value(state.rows(entity), key[0]) if len(key) == 1 else None
|
|
417
|
+
|
|
418
|
+
|
|
423
419
|
def _sample_args(
|
|
424
420
|
operation: Operation, spec, settings, state: WorldState, checker: Checker, documented: dict | None
|
|
425
421
|
) -> tuple[dict[str, str], dict[str, str]]:
|
|
426
422
|
"""Path arguments from the world's own rows — the operation's declared ``sample``
|
|
427
423
|
first (a value a validating handler accepts), then this operation's filter, then any
|
|
428
|
-
operation's filter on the same placeholder, then
|
|
429
|
-
|
|
424
|
+
operation's filter on the same placeholder, then a row of this operation's entity holding a field of that
|
|
425
|
+
name, then the first row of any entity keyed by it, then (for the path's last placeholder only) the entity's
|
|
426
|
+
primary key — else the document's examples. The point is to hit data, not to probe 404s."""
|
|
430
427
|
op = settings.for_operation(operation.name)
|
|
431
428
|
hints = _hints(spec, settings)
|
|
432
429
|
args: dict[str, str] = {}
|
|
@@ -443,9 +440,11 @@ def _sample_args(
|
|
|
443
440
|
entity, path = hints[name]
|
|
444
441
|
value = _first_value(state.rows(entity), path)
|
|
445
442
|
if value is None and operation.entity:
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
443
|
+
value = _first_value(state.rows(operation.entity), name)
|
|
444
|
+
if value is None:
|
|
445
|
+
value = _keyed_value(state, name)
|
|
446
|
+
if value is None and operation.entity and name == list(operation.placeholders)[-1]:
|
|
447
|
+
value = _own_key_value(state, operation.entity)
|
|
449
448
|
if value is None and name in documented_params:
|
|
450
449
|
value = checker.example(documented_params[name].get("schema") or {})
|
|
451
450
|
args[name] = str(value if value is not None else "1")
|
|
@@ -468,8 +467,10 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
|
|
|
468
467
|
state: WorldState = server.world_state # type: ignore[attr-defined]
|
|
469
468
|
outcomes: list[Outcome] = []
|
|
470
469
|
try:
|
|
471
|
-
|
|
472
|
-
|
|
470
|
+
door = Door.of(spec, settings, PIN)
|
|
471
|
+
operations = list(spec.operations.values())[:max_operations]
|
|
472
|
+
operations.sort(key=lambda operation: operation.method == "DELETE")
|
|
473
|
+
for operation in operations:
|
|
473
474
|
located = None
|
|
474
475
|
checker = None
|
|
475
476
|
for document in documents:
|
|
@@ -485,8 +486,8 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
|
|
|
485
486
|
responses = documented.get("responses") or {}
|
|
486
487
|
outcome.expected = [code for code in responses if code.startswith("2")] or ["200"]
|
|
487
488
|
args, params = _sample_args(operation, spec, settings, state, checker, documented)
|
|
488
|
-
|
|
489
|
-
|
|
489
|
+
headers, credential_query = door.present()
|
|
490
|
+
params.update(credential_query)
|
|
490
491
|
path = operation.path
|
|
491
492
|
for name, value in args.items():
|
|
492
493
|
path = path.replace("{" + name + "}", urllib.parse.quote(value, safe=""))
|
|
@@ -516,7 +517,9 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
|
|
|
516
517
|
outcome.status = status
|
|
517
518
|
answer = responses.get(str(status)) or responses.get("default") or {}
|
|
518
519
|
schema = ((answer.get("content") or {}).get("application/json") or {}).get("schema")
|
|
519
|
-
if
|
|
520
|
+
if status in BODILESS_STATUSES and not raw:
|
|
521
|
+
outcome.schema = "no-body"
|
|
522
|
+
elif schema is None:
|
|
520
523
|
outcome.schema = "no-schema"
|
|
521
524
|
else:
|
|
522
525
|
try:
|