@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -16,6 +16,11 @@ or a field named only for what it holds (:data:`NAMED_BY_HOLDER`: ``value``,
16
16
  ``pan``), which elsewhere name other things (``boarding_pass``);
17
17
  * it is a ``number``, ``no`` or ``pan`` under a ``card``.
18
18
 
19
+ Every string kept is scrubbed in place (:meth:`CallRedaction.scrub`), the rest
20
+ of it kept: the session's credential as sent and percent-encoded, the value of a
21
+ query parameter the world's ``[auth]`` carries a secret in (a query key, a signed
22
+ link's ``signature``) wherever a link holding it sits, and a JSON Web Token.
23
+
19
24
  A value is a credential by its shape, whatever its field: a provider key, an
20
25
  AWS key id or a bearer header (:data:`~gatewaysdk.environment.secret_values.SECRET_VALUE`,
21
26
  the browser recorder's rule), the further token shapes of
@@ -30,8 +35,9 @@ Each string or number so named or shaped reads :data:`REDACTED`; the key stays,
30
35
  so a grader can still ask whether it was sent. Booleans, nulls and every other
31
36
  field are kept as sent, so a grader reads what the agent asked, with what
32
37
  no reply can carry read as text: an unpaired surrogate as U+FFFD, a NaN or an
33
- infinity as ``"NaN"``, ``"Infinity"``, ``"-Infinity"``. A query-auth world's own
34
- credential parameter is dropped. A request holding more than
38
+ infinity as ``"NaN"``, ``"Infinity"``, ``"-Infinity"``. A query parameter the
39
+ world's ``[auth]`` carries a secret in (a query-auth key, a signed link's
40
+ signature) is dropped. A request holding more than
35
41
  :data:`MAX_LOGGED_VALUES` values, keys and key parts, or :data:`MAX_LOGGED_CHARS`
36
42
  characters of text, its keys included, is too large to keep
37
43
  (:class:`RequestTooLarge`): its whole request shares one :class:`RequestBudget`.
@@ -43,15 +49,18 @@ import re
43
49
  import urllib.parse
44
50
  from dataclasses import dataclass, replace
45
51
  from functools import cached_property
46
- from typing import Any, Iterator
52
+ from typing import Any, Callable, Iterable, Iterator
47
53
 
48
54
  from ..secret_values import SECRET_TOKEN, SECRET_VALUE, basic_credentials
49
55
  from .connector import PLACEHOLDER, Operation, Spec
56
+ from .door import open_label
50
57
  from .handler_errors import carried_scalar
51
58
 
52
59
  REDACTED = "[redacted]"
53
60
  MAX_LOGGED_VALUES = 100_000
54
61
  MAX_LOGGED_CHARS = 1 << 20
62
+ SCRUB_MARGIN = 8 * 1024
63
+ MIN_SCRUBBED_SECRET = 16
55
64
 
56
65
  CREDENTIAL_WORDS = frozenset(
57
66
  {
@@ -161,7 +170,37 @@ KEPT_SUFFIXES = (
161
170
  "deltatoken",
162
171
  "preauthorization",
163
172
  )
173
+ KEPT_HEADERS = frozenset(
174
+ {
175
+ "accept",
176
+ "cache-control",
177
+ "content-encoding",
178
+ "content-length",
179
+ "content-type",
180
+ "host",
181
+ "idempotency-key",
182
+ "linear-delivery",
183
+ "linear-event",
184
+ "origin",
185
+ "svix-id",
186
+ "svix-timestamp",
187
+ "user-agent",
188
+ "x-github-delivery",
189
+ "x-github-event",
190
+ "x-forwarded-proto",
191
+ "x-request-id",
192
+ }
193
+ )
194
+ KEPT_HEADER_PREFIXES = ("accept-", "if-", "sec-fetch-")
195
+ HEADER_NAME_CHARS = 256
164
196
  _WORD = re.compile(r"[A-Z]?[a-z]+\d*|[A-Z]+\d*(?![a-z])|\d+")
197
+ _QUERY_LEAD = r"(?:[?&;]|%(?:25)?26|%(?:25)?3[Ff]|\\u0026|\\u003[Ff])"
198
+ _QUERY_EQUALS = r"(?:=|%(?:25)?3[Dd]|\\u003[Dd])"
199
+ _QUERY_VALUE = r"[^&#\"'\s\\<>,;)]*"
200
+ _EMBEDDED_TOKEN = re.compile(
201
+ r"(?:(?<![-_A-Za-z0-9])|(?<=%[0-9A-Fa-f]{2})|(?<=\\u00[0-9A-Fa-f]{2}))"
202
+ r"eyJ[-_A-Za-z0-9]{8,}\.eyJ[-_A-Za-z0-9]{8,}\.[-_A-Za-z0-9]+"
203
+ )
165
204
  _CARD_DIGITS = re.compile(r"^\d(?:[ -]?\d){12,18}$")
166
205
 
167
206
 
@@ -194,6 +233,11 @@ def _card_digits(text: str) -> bool:
194
233
  return bool(_CARD_DIGITS.match(text)) and _luhn(re.sub(r"[ -]", "", text))
195
234
 
196
235
 
236
+ def _secret_segment(segment: str, secret: str | None) -> bool:
237
+ decoded = urllib.parse.unquote(segment)
238
+ return _shaped_like_a_secret(decoded) or (bool(secret) and secret in decoded)
239
+
240
+
197
241
  def _shaped_like_a_secret(value: Any) -> bool:
198
242
  if not isinstance(value, str):
199
243
  return False
@@ -269,7 +313,10 @@ class RequestBudget:
269
313
  keys and key parts, and :data:`MAX_LOGGED_CHARS` characters of text, its
270
314
  keys included, an integer counted by its bytes. Each is charged before it is read, so a request past it costs
271
315
  no more than the budget; what it spent is what the logged entry holds, which
272
- the call log charges to its own, per-session budget."""
316
+ the call log charges to its own, per-session budget. It refuses rather than
317
+ fills, so it is never :attr:`full`."""
318
+
319
+ full = False
273
320
 
274
321
  def __init__(self) -> None:
275
322
  self.values = MAX_LOGGED_VALUES
@@ -284,6 +331,43 @@ class RequestBudget:
284
331
  if self.values < 0 or self.chars < 0:
285
332
  raise RequestTooLarge
286
333
 
334
+ def text(self, value: str) -> str:
335
+ return value
336
+
337
+
338
+ class PrefixBudget(RequestBudget):
339
+ """A walk that keeps a value's leading part in the order JSON writes it: it
340
+ never refuses, and once ``size`` values or characters are spent it is
341
+ :attr:`full`, so the walk adds nothing more and records in :attr:`cut` that
342
+ something was left. What it walked writes at least ``size`` characters of
343
+ JSON, so the first ``size`` bytes of the part it kept are the first ``size``
344
+ bytes of the whole."""
345
+
346
+ def __init__(self, size: int) -> None:
347
+ self.size = size
348
+ self.values = size
349
+ self.chars = size
350
+ self.cut = False
351
+
352
+ @property
353
+ def full(self) -> bool:
354
+ return self.values <= 0 or self.chars <= 0
355
+
356
+ def spent(self) -> tuple[int, int]:
357
+ return self.size - self.values, self.size - self.chars
358
+
359
+ def spend(self, values: int, chars: int) -> None:
360
+ self.values -= values
361
+ self.chars -= chars
362
+
363
+ def text(self, value: str) -> str:
364
+ """``value`` to its first ``size`` characters and :data:`SCRUB_MARGIN` more: no
365
+ more of one string can be kept, and the margin holds whole any secret the
366
+ kept part ends inside, so a scrub finds it."""
367
+ kept = value[: self.size + SCRUB_MARGIN]
368
+ self.cut = self.cut or len(kept) < len(value)
369
+ return kept
370
+
287
371
 
288
372
  def _size(value: Any) -> int:
289
373
  """What a scalar holds beyond one value: a string's characters, an integer's bytes."""
@@ -294,42 +378,146 @@ def _size(value: Any) -> int:
294
378
  return 0
295
379
 
296
380
 
297
- def _kept(value: Any, name: FieldName, budget: RequestBudget) -> Any:
381
+ def _kept(value: Any, name: FieldName, budget: RequestBudget, scrub: Callable[[str], str]) -> Any:
298
382
  budget.spend(1, _size(value))
299
383
  if isinstance(value, dict):
300
- return {carried_scalar(key): _kept(item, name.child(key), budget) for key, item in _keys(value, budget)}
384
+ return {carried_scalar(key): _kept(item, name.child(key), budget, scrub) for key, item in _keys(value, budget)}
301
385
  if isinstance(value, list):
302
386
  item_name = name.child("*")
303
- return [_kept(item, item_name, budget) for item in value]
387
+ return [_kept(item, item_name, budget, scrub) for item in _items(value, budget)]
304
388
  if _scalar(value) and name.redacts(value):
305
389
  return REDACTED
306
- return carried_scalar(value)
390
+ return carried_scalar(scrub(budget.text(value)) if isinstance(value, str) else value)
391
+
392
+
393
+ def _spellings(secret: str | None) -> list[str]:
394
+ """``secret`` as it may sit in a body: as sent, and percent-encoded the two ways a URL
395
+ carries it, and the capability label an open session's hostname derives from it; longest
396
+ first, so one spelling never leaves part of another. A secret shorter than
397
+ :data:`MIN_SCRUBBED_SECRET` (a local pin such as ``conform``; a World Host token has 32
398
+ characters) is ordinary text and has none."""
399
+ if not secret or len(secret) < MIN_SCRUBBED_SECRET:
400
+ return []
401
+ forms = {secret, urllib.parse.quote(secret, safe=""), urllib.parse.quote_plus(secret), open_label(secret)}
402
+ return sorted(forms, key=len, reverse=True)
307
403
 
308
404
 
309
405
  def _keys(value: dict, budget: RequestBudget) -> Iterator[tuple[str, Any]]:
310
- for key, item in value.items():
406
+ for key, item in _items(value.items(), budget):
311
407
  text = str(key)
312
408
  budget.spend(text.count(".") + 1, len(text))
313
409
  yield text, item
314
410
 
315
411
 
316
- class CallRedaction:
317
- """The call log's redaction (module docstring), for one world: ``credential_param``
318
- is a query-auth world's credential parameter, dropped from every logged query."""
412
+ def _items(items: Iterable[Any], budget: RequestBudget) -> Iterator[Any]:
413
+ for item in items:
414
+ if budget.full:
415
+ budget.cut = True
416
+ return
417
+ yield item
418
+
319
419
 
320
- def __init__(self, credential_param: str | None = None):
321
- self.credential_param = credential_param
420
+ class CallRedaction:
421
+ """The call log's redaction (module docstring), for one session of one world:
422
+ ``credential_params`` are the query parameters its ``[auth]`` carries a secret in
423
+ (a query-auth world's key, a signed link's signature), dropped from every logged
424
+ query; ``credential_header`` the header its scheme carries a key or a minted token
425
+ in, whatever it is named, kept as :data:`REDACTED` wherever headers are kept;
426
+ ``secret`` the session's credential, which no string kept holds."""
427
+
428
+ def __init__(
429
+ self,
430
+ credential_params: frozenset[str] = frozenset(),
431
+ credential_header: str | None = None,
432
+ secret: str | None = None,
433
+ ):
434
+ self.credential_params = credential_params
435
+ self.credential_header = (credential_header or "").lower()
436
+ self.secret = secret
437
+ self._spellings = _spellings(secret)
438
+ self._names = tuple(sorted(credential_params))
439
+ alternatives = "|".join(re.escape(name) for name in self._names)
440
+ lead = f"({_QUERY_LEAD}(?:{alternatives}){_QUERY_EQUALS}){_QUERY_VALUE}"
441
+ self._carried = re.compile(lead) if alternatives else None
322
442
 
323
443
  @classmethod
324
- def of(cls, spec: Spec) -> CallRedaction:
325
- return cls(spec.auth_name if spec.scheme == "query" else None)
444
+ def of(cls, spec: Spec | None, secret: str | None = None) -> CallRedaction:
445
+ if spec is None:
446
+ return cls(secret=secret)
447
+ return cls(spec.auth.secret_params, spec.auth.scheme.secret_header, secret)
448
+
449
+ def value(self, value: Any, budget: RequestBudget | None = None) -> Any:
450
+ return _kept(value, ROOT, budget or RequestBudget(), self.scrub)
451
+
452
+ def prefix(self, value: Any, size: int) -> tuple[Any, bool]:
453
+ """``value`` redacted as far as its first ``size`` values or characters
454
+ (:class:`PrefixBudget`), and whether any of it was left out."""
455
+ budget = PrefixBudget(size)
456
+ return _kept(value, ROOT, budget, self.scrub), budget.cut
457
+
458
+ def headers(self, headers: dict[str, str], secret: str | None, size: int) -> dict[str, str]:
459
+ """A request's headers as the request ledger keeps them, the first ``size``
460
+ characters of them, every name kept to its first :data:`HEADER_NAME_CHARS`. A
461
+ value is kept only under a name that says what it is (:data:`KEPT_HEADERS`,
462
+ :data:`KEPT_HEADER_PREFIXES`) and then still redacted by its shape; every other
463
+ value, however a vendor names its key, the world's own credential header and any
464
+ value carrying ``secret`` (the session's token) read :data:`REDACTED`."""
465
+ kept, _ = self.prefix(dict(headers), size)
466
+ return {
467
+ name[:HEADER_NAME_CHARS]: self._header(name, headers.get(name, ""), value, secret)
468
+ for name, value in kept.items()
469
+ }
470
+
471
+ def _header(self, name: str, sent: str, kept: Any, secret: str | None) -> Any:
472
+ lowered = name.lower()
473
+ described = lowered in KEPT_HEADERS or lowered.startswith(KEPT_HEADER_PREFIXES)
474
+ carried = bool(secret) and secret in sent
475
+ return kept if described and lowered != self.credential_header and not carried else REDACTED
476
+
477
+ def scrub(self, text: str) -> str:
478
+ """``text`` with every secret it embeds read :data:`REDACTED` in place: the
479
+ session's credential as sent and percent-encoded, an open session's capability
480
+ label (its hostname as a request or a link names it), the value of each query
481
+ parameter the world's ``[auth]`` carries a secret in (in a link a world echoes
482
+ or an agent passes on) and a JSON Web Token. Each is found in time linear in
483
+ ``text``."""
484
+ for spelling in self._spellings:
485
+ text = text.replace(spelling, REDACTED)
486
+ if self._carried is not None and any(name in text for name in self._names):
487
+ text = self._carried.sub(rf"\1{REDACTED}", text)
488
+ return _EMBEDDED_TOKEN.sub(REDACTED, text) if "eyJ" in text else text
326
489
 
327
490
  @staticmethod
328
- def value(value: Any, budget: RequestBudget | None = None) -> Any:
329
- return _kept(value, ROOT, budget or RequestBudget())
491
+ def path_without_secret(path: str, secret: str | None) -> str:
492
+ """``path`` with each segment carrying ``secret`` (the session's token) read :data:`REDACTED`."""
493
+ if not secret:
494
+ return path
495
+ return "/".join(REDACTED if secret in urllib.parse.unquote(part) else part for part in path.split("/"))
496
+
497
+ @staticmethod
498
+ def without_secret(value: Any, secret: str | None) -> Any:
499
+ """``value`` with every string carrying ``secret`` (the session's token), at any
500
+ depth, read :data:`REDACTED`."""
501
+ if not secret:
502
+ return value
503
+ if isinstance(value, str):
504
+ return REDACTED if secret in value else value
505
+ if isinstance(value, dict):
506
+ return {key: CallRedaction.without_secret(item, secret) for key, item in value.items()}
507
+ if isinstance(value, list):
508
+ return [CallRedaction.without_secret(item, secret) for item in value]
509
+ return value
510
+
511
+ def raw_path(self, path: str, secret: str | None) -> str:
512
+ """A path no route matched, so no argument is named: each segment shaped like a
513
+ secret, or carrying ``secret`` (the session's token), reads :data:`REDACTED`,
514
+ and every other one is scrubbed (:meth:`scrub`)."""
515
+ return "/".join(
516
+ REDACTED if _secret_segment(segment, secret) else self.scrub(segment) for segment in path.split("/")
517
+ )
330
518
 
331
519
  def query(self, query_all: dict, budget: RequestBudget | None = None) -> dict:
332
- kept = {k: (v[0] if len(v) == 1 else list(v)) for k, v in query_all.items() if k != self.credential_param}
520
+ kept = {k: (v[0] if len(v) == 1 else list(v)) for k, v in query_all.items() if k not in self.credential_params}
333
521
  return self.value(kept, budget)
334
522
 
335
523
  def path(self, operation: Operation, path: str, args: dict) -> str:
@@ -4,11 +4,11 @@ The TypeScript twin is ``worldData/cast.ts``; ``tests/conformance/csv_cast.json`
4
4
  them to the same answers. ``integer`` and ``number`` parse as numbers (a number is
5
5
  always a float), ``boolean`` takes true/false/1/0 in any case, ``array`` and
6
6
  ``object`` parse as JSON of that shape, ``string`` is the text as given. An empty
7
- cell is ``None`` when the field's type admits null, left out when the field is
8
- optional, the empty string for a required string, and refused for any other
9
- required field. A union type (``integer | string``) tries each member in order,
10
- ``string`` last. A cell that fits none is refused, naming the field and the cell;
11
- a column the schema does not name stays text (the gate refuses it, as it always did).
7
+ cell is ``None`` when the field's type admits null, the empty string for a required
8
+ string, and left out otherwise. A union type (``integer | string``) tries each member
9
+ in order, ``string`` last. A cell that fits none stays its text, as does a column the
10
+ schema does not name: the contract's check then refuses the row, naming its entity,
11
+ key and field, as it refuses a JSON row, so a file reports every bad row at once.
12
12
 
13
13
  Standard library only, so the bridge and the api bundle can load it.
14
14
  """
@@ -23,8 +23,6 @@ from dataclasses import dataclass
23
23
  from pathlib import Path
24
24
  from typing import Any
25
25
 
26
- from .validation import fail
27
-
28
26
  INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
29
27
  NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
30
28
  BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
@@ -51,21 +49,6 @@ class FieldType:
51
49
  return described
52
50
 
53
51
 
54
- def _article(kind: str) -> str:
55
- return "an" if kind[:1].lower() in "aeiou" else "a"
56
-
57
-
58
- def refuse(where: str, field: str, cell: str | None, kind: str) -> None:
59
- """The row-level refusal a cell that will not cast earns (same words as the TypeScript twin)."""
60
- if cell is None:
61
- fail("rows", f"{where}: {field} is empty; the field is a required {kind}", where)
62
- fail(
63
- "rows",
64
- f"{where}: {field} = {json.dumps(cell)} is not {_article(kind)} {kind}",
65
- where,
66
- )
67
-
68
-
69
52
  def type_members(kind: str | None) -> list[str]:
70
53
  """The typed members of a field's ``type`` (``"integer | null"`` -> integer, null); an
71
54
  unknown word (a ``$ref`` name) is dropped."""
@@ -123,10 +106,11 @@ class FieldCaster:
123
106
  self.required = required
124
107
  self.members = type_members(kind)
125
108
 
126
- def cast(self, cell: str | None, where: str) -> Any:
127
- """The typed value; :data:`OMIT` to leave the field out of the row."""
109
+ def cast(self, cell: str | None) -> Any:
110
+ """The typed value; :data:`OMIT` to leave the field out of the row; the cell as given when
111
+ it fits no member, for the contract's check to refuse."""
128
112
  if cell is None:
129
- return self._empty(where)
113
+ return self._empty()
130
114
  if not self.members:
131
115
  return cell
132
116
  text = cell.strip()
@@ -134,18 +118,14 @@ class FieldCaster:
134
118
  value = _CAST.get(member, _no_cast)(text)
135
119
  if value is not _MISSING:
136
120
  return value
137
- if "string" in self.members:
138
- return cell
139
- refuse(where, self.name, cell, self.type or "value")
121
+ return cell
140
122
 
141
- def _empty(self, where: str) -> Any:
123
+ def _empty(self) -> Any:
142
124
  if not self.members or "null" in self.members:
143
125
  return None
144
- if not self.required:
145
- return OMIT
146
- if "string" in self.members:
126
+ if self.required and "string" in self.members:
147
127
  return ""
148
- refuse(where, self.name, None, self.type or "value")
128
+ return OMIT
149
129
 
150
130
 
151
131
  #: What :meth:`FieldCaster.cast` answers for an empty optional cell: the field stays out of the row.
@@ -158,11 +138,11 @@ class RowCaster:
158
138
  def __init__(self, fields: Iterable[FieldType]):
159
139
  self._fields = {f.name: FieldCaster(f.name, f.type, f.required) for f in fields}
160
140
 
161
- def row(self, record: dict[str, str | None], where: str) -> dict:
141
+ def row(self, record: dict[str, str | None]) -> dict:
162
142
  row: dict = {}
163
143
  for name, cell in record.items():
164
144
  caster = self._fields.get(name)
165
- value = cell if caster is None else caster.cast(cell, where)
145
+ value = cell if caster is None else caster.cast(cell)
166
146
  if value is not OMIT:
167
147
  row[name] = value
168
148
  return row
@@ -899,7 +899,10 @@ def bind_connection(bundle: Path, contract: Contract) -> None:
899
899
  return
900
900
  from .connector import parse_spec
901
901
 
902
- parse_spec(spec_path.read_text(encoding="utf-8")).check_entities(contract.entities)
902
+ spec = parse_spec(spec_path.read_text(encoding="utf-8"))
903
+ spec.check_entities(contract.entities)
904
+ if spec.webhooks is not None:
905
+ spec.webhooks.check_contract(contract.entities)
903
906
 
904
907
 
905
908
  def check_bundle(bundle: str | Path) -> Contract:
@@ -26,7 +26,6 @@ body the document's example does not supply.
26
26
 
27
27
  from __future__ import annotations
28
28
 
29
- import base64
30
29
  import json
31
30
  import threading
32
31
  import urllib.error
@@ -34,16 +33,20 @@ import urllib.parse
34
33
  import urllib.request
35
34
  from dataclasses import dataclass, field
36
35
  from pathlib import Path
37
- from typing import Any
36
+ from typing import Any, Literal
38
37
 
39
38
  from .._toml import tomllib
40
39
  from .api import WorldState, load_api_settings, serve
41
40
  from .connector import SPEC, Operation, load_spec
41
+ from .door import Door
42
42
  from .validation import fail
43
43
 
44
44
  PIN = "conform"
45
45
 
46
46
 
47
+ BODILESS_STATUSES = (204, 205)
48
+
49
+
47
50
  def load_openapi(path: str | Path) -> dict:
48
51
  return parse_openapi(Path(path).read_text(encoding="utf-8"), str(path))
49
52
 
@@ -306,7 +309,7 @@ class Outcome:
306
309
  documented: bool
307
310
  status: int | None = None
308
311
  expected: list[str] = field(default_factory=list)
309
- schema: str = "unchecked" # ok | violations | no-schema | unchecked
312
+ schema: Literal["ok", "violations", "no-schema", "unchecked"] = "unchecked"
310
313
  problems: list[str] = field(default_factory=list)
311
314
  request: dict = field(default_factory=dict)
312
315
 
@@ -316,7 +319,7 @@ class Outcome:
316
319
  self.documented
317
320
  and self.status is not None
318
321
  and str(self.status) in self.expected
319
- and self.schema in ("ok", "no-schema")
322
+ and self.schema in ("ok", "no-schema", "no-body")
320
323
  )
321
324
 
322
325
 
@@ -341,27 +344,6 @@ class Report:
341
344
  return {"world": self.world, "summary": self.summary, "operations": [o.__dict__ for o in self.outcomes]}
342
345
 
343
346
 
344
- def _credentials(base: str, spec, settings) -> dict[str, str]:
345
- """Headers that get the conformance run through the world's own auth, using the pin."""
346
- if spec.scheme == "none":
347
- return {}
348
- if spec.scheme == "header":
349
- return {spec.auth_name or "": PIN}
350
- if spec.scheme == "bearer":
351
- return {"Authorization": f"Bearer {PIN}"}
352
- if spec.scheme == "basic":
353
- return {"Authorization": "Basic " + base64.b64encode(PIN.encode()).decode()}
354
- if spec.scheme == "query":
355
- return {}
356
- form = urllib.parse.urlencode(
357
- {"grant_type": "client_credentials", "client_id": "conform", "client_secret": PIN}
358
- ).encode()
359
- prefix = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
360
- request = urllib.request.Request(base + prefix + (settings.token_path or "/oauth/token"), data=form, method="POST")
361
- with urllib.request.urlopen(request, timeout=10) as reply:
362
- return {"Authorization": "Bearer " + json.loads(reply.read())["access_token"]}
363
-
364
-
365
347
  def request_samples(world: str | Path) -> dict[str, tuple[dict[str, str], Any]]:
366
348
  """Per operation, the ``[operations.api] sample_headers`` (header -> value) and
367
349
  ``sample_body`` (a table or array, sent as JSON) declared in ``connector.toml``.
@@ -420,13 +402,28 @@ def _first_value(rows: list[dict], path: str) -> Any:
420
402
  return None
421
403
 
422
404
 
405
+ def _keyed_value(state: WorldState, name: str) -> Any:
406
+ for entity, declaration in state.contract.entities.items():
407
+ if name in declaration["primary_key"]:
408
+ value = _first_value(state.rows(entity), name)
409
+ if value is not None:
410
+ return value
411
+ return None
412
+
413
+
414
+ def _own_key_value(state: WorldState, entity: str) -> Any:
415
+ key = state.contract.entities[entity]["primary_key"]
416
+ return _first_value(state.rows(entity), key[0]) if len(key) == 1 else None
417
+
418
+
423
419
  def _sample_args(
424
420
  operation: Operation, spec, settings, state: WorldState, checker: Checker, documented: dict | None
425
421
  ) -> tuple[dict[str, str], dict[str, str]]:
426
422
  """Path arguments from the world's own rows — the operation's declared ``sample``
427
423
  first (a value a validating handler accepts), then this operation's filter, then any
428
- operation's filter on the same placeholder, then the entity's primary key — else the
429
- document's examples. The point is to hit data, not to probe 404s."""
424
+ operation's filter on the same placeholder, then a row of this operation's entity holding a field of that
425
+ name, then the first row of any entity keyed by it, then (for the path's last placeholder only) the entity's
426
+ primary key — else the document's examples. The point is to hit data, not to probe 404s."""
430
427
  op = settings.for_operation(operation.name)
431
428
  hints = _hints(spec, settings)
432
429
  args: dict[str, str] = {}
@@ -443,9 +440,11 @@ def _sample_args(
443
440
  entity, path = hints[name]
444
441
  value = _first_value(state.rows(entity), path)
445
442
  if value is None and operation.entity:
446
- key = state.contract.entities[operation.entity]["primary_key"]
447
- if len(key) == 1:
448
- value = _first_value(state.rows(operation.entity), key[0])
443
+ value = _first_value(state.rows(operation.entity), name)
444
+ if value is None:
445
+ value = _keyed_value(state, name)
446
+ if value is None and operation.entity and name == list(operation.placeholders)[-1]:
447
+ value = _own_key_value(state, operation.entity)
449
448
  if value is None and name in documented_params:
450
449
  value = checker.example(documented_params[name].get("schema") or {})
451
450
  args[name] = str(value if value is not None else "1")
@@ -468,8 +467,10 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
468
467
  state: WorldState = server.world_state # type: ignore[attr-defined]
469
468
  outcomes: list[Outcome] = []
470
469
  try:
471
- headers = _credentials(base, spec, settings)
472
- for operation in list(spec.operations.values())[:max_operations]:
470
+ door = Door.of(spec, settings, PIN)
471
+ operations = list(spec.operations.values())[:max_operations]
472
+ operations.sort(key=lambda operation: operation.method == "DELETE")
473
+ for operation in operations:
473
474
  located = None
474
475
  checker = None
475
476
  for document in documents:
@@ -485,8 +486,8 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
485
486
  responses = documented.get("responses") or {}
486
487
  outcome.expected = [code for code in responses if code.startswith("2")] or ["200"]
487
488
  args, params = _sample_args(operation, spec, settings, state, checker, documented)
488
- if spec.scheme == "query":
489
- params[spec.auth_name or ""] = PIN
489
+ headers, credential_query = door.present()
490
+ params.update(credential_query)
490
491
  path = operation.path
491
492
  for name, value in args.items():
492
493
  path = path.replace("{" + name + "}", urllib.parse.quote(value, safe=""))
@@ -516,7 +517,9 @@ def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int
516
517
  outcome.status = status
517
518
  answer = responses.get(str(status)) or responses.get("default") or {}
518
519
  schema = ((answer.get("content") or {}).get("application/json") or {}).get("schema")
519
- if schema is None:
520
+ if status in BODILESS_STATUSES and not raw:
521
+ outcome.schema = "no-body"
522
+ elif schema is None:
520
523
  outcome.schema = "no-schema"
521
524
  else:
522
525
  try: