@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,526 @@
1
+ """What the call log keeps of a request: everything but a credential.
2
+
3
+ A field is a credential by its dotted path (``card.number``, ``cards.*.number``,
4
+ ``newPassword``), its words split at case changes and separators with trailing
5
+ digits dropped (``password2``) and a plural parent read singular. A list item,
6
+ or a field named only for what it holds (:data:`NAMED_BY_HOLDER`: ``value``,
7
+ ``answer``, ``new``...), is named by the field it sits in, read singular
8
+ (``passwords.*``, Okta's ``credentials.password.value``):
9
+
10
+ * its last word is one of :data:`CREDENTIAL_WORDS`, or ends in one of
11
+ :data:`CREDENTIAL_SUFFIXES` (``newpassword``, ``accesstoken``);
12
+ * its last two words are one of :data:`CREDENTIAL_PAIRS`, or its last word ends
13
+ in one joined (``cardnumber``, ``licensekey``), or its holder's last word and
14
+ its own key are one (``security.answer``);
15
+ * the whole key is one of :data:`WHOLE_KEY_CREDENTIALS` (``pass``, ``auth``,
16
+ ``pan``), which elsewhere name other things (``boarding_pass``);
17
+ * it is a ``number``, ``no`` or ``pan`` under a ``card``.
18
+
19
+ Every string kept is scrubbed in place (:meth:`CallRedaction.scrub`), the rest
20
+ of it kept: the session's credential as sent and percent-encoded, the value of a
21
+ query parameter the world's ``[auth]`` carries a secret in (a query key, a signed
22
+ link's ``signature``) wherever a link holding it sits, and a JSON Web Token.
23
+
24
+ A value is a credential by its shape, whatever its field: a provider key, an
25
+ AWS key id or a bearer header (:data:`~gatewaysdk.environment.secret_values.SECRET_VALUE`,
26
+ the browser recorder's rule), the further token shapes of
27
+ :data:`~gatewaysdk.environment.secret_values.SECRET_TOKEN`, a Basic
28
+ authorization value, and a Luhn-valid 13 to 19 digit number in a field named
29
+ only one of :data:`LUHN_KEYS` (an ``order_number`` is not one). Pagination
30
+ cursors (``pageToken``, ``nextToken``, ``syncToken``, Graph's ``$skiptoken``
31
+ and ``$deltatoken``) and NetSuite's payment authorizations (``amountPendingAuthorization``)
32
+ are kept (:data:`KEPT_PAIRS`, :data:`KEPT_SUFFIXES`).
33
+
34
+ Each string or number so named or shaped reads :data:`REDACTED`; the key stays,
35
+ so a grader can still ask whether it was sent. Booleans, nulls and every other
36
+ field are kept as sent, so a grader reads what the agent asked, with what
37
+ no reply can carry read as text: an unpaired surrogate as U+FFFD, a NaN or an
38
+ infinity as ``"NaN"``, ``"Infinity"``, ``"-Infinity"``. A query parameter the
39
+ world's ``[auth]`` carries a secret in (a query-auth key, a signed link's
40
+ signature) is dropped. A request holding more than
41
+ :data:`MAX_LOGGED_VALUES` values, keys and key parts, or :data:`MAX_LOGGED_CHARS`
42
+ characters of text, its keys included, is too large to keep
43
+ (:class:`RequestTooLarge`): its whole request shares one :class:`RequestBudget`.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import re
49
+ import urllib.parse
50
+ from dataclasses import dataclass, replace
51
+ from functools import cached_property
52
+ from typing import Any, Callable, Iterable, Iterator
53
+
54
+ from ..secret_values import SECRET_TOKEN, SECRET_VALUE, basic_credentials
55
+ from .connector import PLACEHOLDER, Operation, Spec
56
+ from .door import open_label
57
+ from .handler_errors import carried_scalar
58
+
59
+ REDACTED = "[redacted]"
60
+ MAX_LOGGED_VALUES = 100_000
61
+ MAX_LOGGED_CHARS = 1 << 20
62
+ SCRUB_MARGIN = 8 * 1024
63
+ MIN_SCRUBBED_SECRET = 16
64
+
65
+ CREDENTIAL_WORDS = frozenset(
66
+ {
67
+ "password",
68
+ "passwd",
69
+ "passphrase",
70
+ "passcode",
71
+ "pwd",
72
+ "pw",
73
+ "secret",
74
+ "token",
75
+ "apikey",
76
+ "credential",
77
+ "credentials",
78
+ "authorization",
79
+ "bearer",
80
+ "jwt",
81
+ "cookie",
82
+ "pin",
83
+ "otp",
84
+ "totp",
85
+ "cvv",
86
+ "cvc",
87
+ "csc",
88
+ "ssn",
89
+ "esig",
90
+ "esign",
91
+ "esignature",
92
+ }
93
+ )
94
+ CREDENTIAL_SUFFIXES = (
95
+ "password",
96
+ "passwd",
97
+ "passphrase",
98
+ "passcode",
99
+ "secret",
100
+ "token",
101
+ "apikey",
102
+ "credential",
103
+ "credentials",
104
+ "authorization",
105
+ "cookie",
106
+ "esignature",
107
+ )
108
+ CREDENTIAL_PAIRS = frozenset(
109
+ {
110
+ ("api", "key"),
111
+ ("private", "key"),
112
+ ("secret", "key"),
113
+ ("access", "key"),
114
+ ("license", "key"),
115
+ ("encryption", "key"),
116
+ ("signing", "key"),
117
+ ("card", "number"),
118
+ ("card", "no"),
119
+ ("card", "num"),
120
+ ("cc", "number"),
121
+ ("cc", "num"),
122
+ ("pan", "number"),
123
+ ("pan", "no"),
124
+ ("credit", "card"),
125
+ ("security", "code"),
126
+ ("security", "answer"),
127
+ ("security", "number"),
128
+ ("security", "question"),
129
+ ("recovery", "answer"),
130
+ ("recovery", "question"),
131
+ ("pin", "code"),
132
+ ("pin", "number"),
133
+ ("pin", "block"),
134
+ ("pass", "code"),
135
+ ("pass", "phrase"),
136
+ ("otp", "code"),
137
+ ("mfa", "code"),
138
+ ("auth", "code"),
139
+ ("authorization", "code"),
140
+ ("verification", "code"),
141
+ ("password", "confirm"),
142
+ ("password", "confirmation"),
143
+ ("e", "sig"),
144
+ ("e", "sign"),
145
+ ("e", "signature"),
146
+ ("electronic", "signature"),
147
+ }
148
+ )
149
+ JOINED_PAIRS = tuple(first + second for first, second in CREDENTIAL_PAIRS)
150
+ WHOLE_KEY_CREDENTIALS = frozenset({"pass", "auth", "pan"})
151
+ CARD_FIELDS = frozenset({"number", "no", "pan"})
152
+ LUHN_KEYS = frozenset({"number", "no", "num", "card", "cc"})
153
+ NAMED_BY_HOLDER = frozenset({"*", "value", "answer", "current", "new", "old", "previous", "confirm", "confirmation"})
154
+ KEPT_PAIRS = frozenset(
155
+ {
156
+ ("page", "token"),
157
+ ("next", "token"),
158
+ ("sync", "token"),
159
+ ("continuation", "token"),
160
+ ("pending", "authorization"),
161
+ ("payment", "authorization"),
162
+ }
163
+ )
164
+ KEPT_SUFFIXES = (
165
+ "pagetoken",
166
+ "nexttoken",
167
+ "synctoken",
168
+ "continuationtoken",
169
+ "skiptoken",
170
+ "deltatoken",
171
+ "preauthorization",
172
+ )
173
+ KEPT_HEADERS = frozenset(
174
+ {
175
+ "accept",
176
+ "cache-control",
177
+ "content-encoding",
178
+ "content-length",
179
+ "content-type",
180
+ "host",
181
+ "idempotency-key",
182
+ "origin",
183
+ "user-agent",
184
+ "x-forwarded-proto",
185
+ "x-request-id",
186
+ }
187
+ )
188
+ KEPT_HEADER_PREFIXES = ("accept-", "if-", "sec-fetch-")
189
+ HEADER_NAME_CHARS = 256
190
+ _WORD = re.compile(r"[A-Z]?[a-z]+\d*|[A-Z]+\d*(?![a-z])|\d+")
191
+ _QUERY_LEAD = r"(?:[?&;]|%(?:25)?26|%(?:25)?3[Ff]|\\u0026|\\u003[Ff])"
192
+ _QUERY_EQUALS = r"(?:=|%(?:25)?3[Dd]|\\u003[Dd])"
193
+ _QUERY_VALUE = r"[^&#\"'\s\\<>,;)]*"
194
+ _EMBEDDED_TOKEN = re.compile(
195
+ r"(?:(?<![-_A-Za-z0-9])|(?<=%[0-9A-Fa-f]{2})|(?<=\\u00[0-9A-Fa-f]{2}))"
196
+ r"eyJ[-_A-Za-z0-9]{8,}\.eyJ[-_A-Za-z0-9]{8,}\.[-_A-Za-z0-9]+"
197
+ )
198
+ _CARD_DIGITS = re.compile(r"^\d(?:[ -]?\d){12,18}$")
199
+
200
+
201
+ class RequestTooLarge(Exception):
202
+ """A request past the call log's budget: logged without its arguments."""
203
+
204
+
205
+ def _singular(word: str) -> str:
206
+ return word[:-1] if len(word) > 3 and word.endswith("s") and not word.endswith("ss") else word
207
+
208
+
209
+ def _words(field: str) -> list[str]:
210
+ stripped = (word.lower().rstrip("0123456789") for word in _WORD.findall(field))
211
+ return [word for word in stripped if word]
212
+
213
+
214
+ def _scalar(value: Any) -> bool:
215
+ return isinstance(value, (str, int, float)) and not isinstance(value, bool)
216
+
217
+
218
+ def _luhn(digits: str) -> bool:
219
+ total = 0
220
+ for index, digit in enumerate(reversed(digits)):
221
+ doubled = int(digit) * (2 if index % 2 else 1)
222
+ total += doubled - 9 if doubled > 9 else doubled
223
+ return total % 10 == 0
224
+
225
+
226
+ def _card_digits(text: str) -> bool:
227
+ return bool(_CARD_DIGITS.match(text)) and _luhn(re.sub(r"[ -]", "", text))
228
+
229
+
230
+ def _secret_segment(segment: str, secret: str | None) -> bool:
231
+ decoded = urllib.parse.unquote(segment)
232
+ return _shaped_like_a_secret(decoded) or (bool(secret) and secret in decoded)
233
+
234
+
235
+ def _shaped_like_a_secret(value: Any) -> bool:
236
+ if not isinstance(value, str):
237
+ return False
238
+ return bool(SECRET_VALUE.match(value) or SECRET_TOKEN.search(value)) or basic_credentials(value)
239
+
240
+
241
+ @dataclass(frozen=True)
242
+ class FieldName:
243
+ """What a field's dotted path says about it (module docstring), built one
244
+ key at a time so a request is read in time linear in its size: the field's
245
+ last word and the word before it, whether a ``card`` comes earlier, its own
246
+ key, and the key it is held under when it is a list item or a field named
247
+ only for what it holds."""
248
+
249
+ last: str = ""
250
+ prior: str = ""
251
+ under_card: bool = False
252
+ segment: str = ""
253
+ held: str = ""
254
+
255
+ def child(self, key: str) -> FieldName:
256
+ name = self
257
+ for part in key.split("."):
258
+ name = name._part(part)
259
+ return name
260
+
261
+ def _part(self, part: str) -> FieldName:
262
+ lowered = part.lower()
263
+ if self.segment and lowered in NAMED_BY_HOLDER:
264
+ return replace(self, held=lowered)
265
+ words = _words(part)
266
+ if not words:
267
+ return replace(self, segment=lowered, held="")
268
+ earlier = [self.last, *words[:-1]] if self.last else words[:-1]
269
+ return FieldName(
270
+ last=words[-1],
271
+ prior=_singular(earlier[-1]) if earlier else "",
272
+ under_card=self.under_card or any(_singular(word) == "card" for word in earlier),
273
+ segment=lowered,
274
+ )
275
+
276
+ @cached_property
277
+ def credential(self) -> bool:
278
+ last = _singular(self.last) if self.held else self.last
279
+ pair = (self.prior, last)
280
+ if not last or pair in KEPT_PAIRS or last.endswith(KEPT_SUFFIXES):
281
+ return False
282
+ return (
283
+ last in CREDENTIAL_WORDS
284
+ or last.endswith(CREDENTIAL_SUFFIXES)
285
+ or pair in CREDENTIAL_PAIRS
286
+ or (last, self.held) in CREDENTIAL_PAIRS
287
+ or last.endswith(JOINED_PAIRS)
288
+ or self.segment in WHOLE_KEY_CREDENTIALS
289
+ or (last in CARD_FIELDS and self.under_card)
290
+ )
291
+
292
+ @cached_property
293
+ def holds_a_card_number(self) -> bool:
294
+ return (_singular(self.segment) if self.held else self.segment) in LUHN_KEYS
295
+
296
+ def redacts(self, value: Any) -> bool:
297
+ if self.credential or _shaped_like_a_secret(value):
298
+ return True
299
+ return self.holds_a_card_number and _card_digits(str(value))
300
+
301
+
302
+ ROOT = FieldName()
303
+
304
+
305
+ class RequestBudget:
306
+ """What one request may cost the call log: :data:`MAX_LOGGED_VALUES` values,
307
+ keys and key parts, and :data:`MAX_LOGGED_CHARS` characters of text, its
308
+ keys included, an integer counted by its bytes. Each is charged before it is read, so a request past it costs
309
+ no more than the budget; what it spent is what the logged entry holds, which
310
+ the call log charges to its own, per-session budget. It refuses rather than
311
+ fills, so it is never :attr:`full`."""
312
+
313
+ full = False
314
+
315
+ def __init__(self) -> None:
316
+ self.values = MAX_LOGGED_VALUES
317
+ self.chars = MAX_LOGGED_CHARS
318
+
319
+ def spent(self) -> tuple[int, int]:
320
+ return MAX_LOGGED_VALUES - self.values, MAX_LOGGED_CHARS - self.chars
321
+
322
+ def spend(self, values: int, chars: int) -> None:
323
+ self.values -= values
324
+ self.chars -= chars
325
+ if self.values < 0 or self.chars < 0:
326
+ raise RequestTooLarge
327
+
328
+ def text(self, value: str) -> str:
329
+ return value
330
+
331
+
332
+ class PrefixBudget(RequestBudget):
333
+ """A walk that keeps a value's leading part in the order JSON writes it: it
334
+ never refuses, and once ``size`` values or characters are spent it is
335
+ :attr:`full`, so the walk adds nothing more and records in :attr:`cut` that
336
+ something was left. What it walked writes at least ``size`` characters of
337
+ JSON, so the first ``size`` bytes of the part it kept are the first ``size``
338
+ bytes of the whole."""
339
+
340
+ def __init__(self, size: int) -> None:
341
+ self.size = size
342
+ self.values = size
343
+ self.chars = size
344
+ self.cut = False
345
+
346
+ @property
347
+ def full(self) -> bool:
348
+ return self.values <= 0 or self.chars <= 0
349
+
350
+ def spent(self) -> tuple[int, int]:
351
+ return self.size - self.values, self.size - self.chars
352
+
353
+ def spend(self, values: int, chars: int) -> None:
354
+ self.values -= values
355
+ self.chars -= chars
356
+
357
+ def text(self, value: str) -> str:
358
+ """``value`` to its first ``size`` characters and :data:`SCRUB_MARGIN` more: no
359
+ more of one string can be kept, and the margin holds whole any secret the
360
+ kept part ends inside, so a scrub finds it."""
361
+ kept = value[: self.size + SCRUB_MARGIN]
362
+ self.cut = self.cut or len(kept) < len(value)
363
+ return kept
364
+
365
+
366
+ def _size(value: Any) -> int:
367
+ """What a scalar holds beyond one value: a string's characters, an integer's bytes."""
368
+ if isinstance(value, str):
369
+ return len(value)
370
+ if isinstance(value, int) and not isinstance(value, bool):
371
+ return (value.bit_length() + 7) // 8
372
+ return 0
373
+
374
+
375
+ def _kept(value: Any, name: FieldName, budget: RequestBudget, scrub: Callable[[str], str]) -> Any:
376
+ budget.spend(1, _size(value))
377
+ if isinstance(value, dict):
378
+ return {carried_scalar(key): _kept(item, name.child(key), budget, scrub) for key, item in _keys(value, budget)}
379
+ if isinstance(value, list):
380
+ item_name = name.child("*")
381
+ return [_kept(item, item_name, budget, scrub) for item in _items(value, budget)]
382
+ if _scalar(value) and name.redacts(value):
383
+ return REDACTED
384
+ return carried_scalar(scrub(budget.text(value)) if isinstance(value, str) else value)
385
+
386
+
387
+ def _spellings(secret: str | None) -> list[str]:
388
+ """``secret`` as it may sit in a body: as sent, and percent-encoded the two ways a URL
389
+ carries it, and the capability label an open session's hostname derives from it; longest
390
+ first, so one spelling never leaves part of another. A secret shorter than
391
+ :data:`MIN_SCRUBBED_SECRET` (a local pin such as ``conform``; a World Host token has 32
392
+ characters) is ordinary text and has none."""
393
+ if not secret or len(secret) < MIN_SCRUBBED_SECRET:
394
+ return []
395
+ forms = {secret, urllib.parse.quote(secret, safe=""), urllib.parse.quote_plus(secret), open_label(secret)}
396
+ return sorted(forms, key=len, reverse=True)
397
+
398
+
399
+ def _keys(value: dict, budget: RequestBudget) -> Iterator[tuple[str, Any]]:
400
+ for key, item in _items(value.items(), budget):
401
+ text = str(key)
402
+ budget.spend(text.count(".") + 1, len(text))
403
+ yield text, item
404
+
405
+
406
+ def _items(items: Iterable[Any], budget: RequestBudget) -> Iterator[Any]:
407
+ for item in items:
408
+ if budget.full:
409
+ budget.cut = True
410
+ return
411
+ yield item
412
+
413
+
414
+ class CallRedaction:
415
+ """The call log's redaction (module docstring), for one session of one world:
416
+ ``credential_params`` are the query parameters its ``[auth]`` carries a secret in
417
+ (a query-auth world's key, a signed link's signature), dropped from every logged
418
+ query; ``credential_header`` the header its scheme carries a key or a minted token
419
+ in, whatever it is named, kept as :data:`REDACTED` wherever headers are kept;
420
+ ``secret`` the session's credential, which no string kept holds."""
421
+
422
+ def __init__(
423
+ self,
424
+ credential_params: frozenset[str] = frozenset(),
425
+ credential_header: str | None = None,
426
+ secret: str | None = None,
427
+ ):
428
+ self.credential_params = credential_params
429
+ self.credential_header = (credential_header or "").lower()
430
+ self.secret = secret
431
+ self._spellings = _spellings(secret)
432
+ self._names = tuple(sorted(credential_params))
433
+ alternatives = "|".join(re.escape(name) for name in self._names)
434
+ lead = f"({_QUERY_LEAD}(?:{alternatives}){_QUERY_EQUALS}){_QUERY_VALUE}"
435
+ self._carried = re.compile(lead) if alternatives else None
436
+
437
+ @classmethod
438
+ def of(cls, spec: Spec | None, secret: str | None = None) -> CallRedaction:
439
+ if spec is None:
440
+ return cls(secret=secret)
441
+ return cls(spec.auth.secret_params, spec.auth.scheme.secret_header, secret)
442
+
443
+ def value(self, value: Any, budget: RequestBudget | None = None) -> Any:
444
+ return _kept(value, ROOT, budget or RequestBudget(), self.scrub)
445
+
446
+ def prefix(self, value: Any, size: int) -> tuple[Any, bool]:
447
+ """``value`` redacted as far as its first ``size`` values or characters
448
+ (:class:`PrefixBudget`), and whether any of it was left out."""
449
+ budget = PrefixBudget(size)
450
+ return _kept(value, ROOT, budget, self.scrub), budget.cut
451
+
452
+ def headers(self, headers: dict[str, str], secret: str | None, size: int) -> dict[str, str]:
453
+ """A request's headers as the request ledger keeps them, the first ``size``
454
+ characters of them, every name kept to its first :data:`HEADER_NAME_CHARS`. A
455
+ value is kept only under a name that says what it is (:data:`KEPT_HEADERS`,
456
+ :data:`KEPT_HEADER_PREFIXES`) and then still redacted by its shape; every other
457
+ value, however a vendor names its key, the world's own credential header and any
458
+ value carrying ``secret`` (the session's token) read :data:`REDACTED`."""
459
+ kept, _ = self.prefix(dict(headers), size)
460
+ return {
461
+ name[:HEADER_NAME_CHARS]: self._header(name, headers.get(name, ""), value, secret)
462
+ for name, value in kept.items()
463
+ }
464
+
465
+ def _header(self, name: str, sent: str, kept: Any, secret: str | None) -> Any:
466
+ lowered = name.lower()
467
+ described = lowered in KEPT_HEADERS or lowered.startswith(KEPT_HEADER_PREFIXES)
468
+ carried = bool(secret) and secret in sent
469
+ return kept if described and lowered != self.credential_header and not carried else REDACTED
470
+
471
+ def scrub(self, text: str) -> str:
472
+ """``text`` with every secret it embeds read :data:`REDACTED` in place: the
473
+ session's credential as sent and percent-encoded, an open session's capability
474
+ label (its hostname as a request or a link names it), the value of each query
475
+ parameter the world's ``[auth]`` carries a secret in (in a link a world echoes
476
+ or an agent passes on) and a JSON Web Token. Each is found in time linear in
477
+ ``text``."""
478
+ for spelling in self._spellings:
479
+ text = text.replace(spelling, REDACTED)
480
+ if self._carried is not None and any(name in text for name in self._names):
481
+ text = self._carried.sub(rf"\1{REDACTED}", text)
482
+ return _EMBEDDED_TOKEN.sub(REDACTED, text) if "eyJ" in text else text
483
+
484
+ @staticmethod
485
+ def path_without_secret(path: str, secret: str | None) -> str:
486
+ """``path`` with each segment carrying ``secret`` (the session's token) read :data:`REDACTED`."""
487
+ if not secret:
488
+ return path
489
+ return "/".join(REDACTED if secret in urllib.parse.unquote(part) else part for part in path.split("/"))
490
+
491
+ @staticmethod
492
+ def without_secret(value: Any, secret: str | None) -> Any:
493
+ """``value`` with every string carrying ``secret`` (the session's token), at any
494
+ depth, read :data:`REDACTED`."""
495
+ if not secret:
496
+ return value
497
+ if isinstance(value, str):
498
+ return REDACTED if secret in value else value
499
+ if isinstance(value, dict):
500
+ return {key: CallRedaction.without_secret(item, secret) for key, item in value.items()}
501
+ if isinstance(value, list):
502
+ return [CallRedaction.without_secret(item, secret) for item in value]
503
+ return value
504
+
505
+ def raw_path(self, path: str, secret: str | None) -> str:
506
+ """A path no route matched, so no argument is named: each segment shaped like a
507
+ secret, or carrying ``secret`` (the session's token), reads :data:`REDACTED`,
508
+ and every other one is scrubbed (:meth:`scrub`)."""
509
+ return "/".join(
510
+ REDACTED if _secret_segment(segment, secret) else self.scrub(segment) for segment in path.split("/")
511
+ )
512
+
513
+ def query(self, query_all: dict, budget: RequestBudget | None = None) -> dict:
514
+ kept = {k: (v[0] if len(v) == 1 else list(v)) for k, v in query_all.items() if k not in self.credential_params}
515
+ return self.value(kept, budget)
516
+
517
+ def path(self, operation: Operation, path: str, args: dict) -> str:
518
+ """The request path as sent, or, when a path argument is a credential,
519
+ the operation's path with that segment :data:`REDACTED`."""
520
+ if not any(self.value({name: value}) != {name: value} for name, value in args.items()):
521
+ return path
522
+ return PLACEHOLDER.sub(lambda found: self._segment(found.group(1), args), operation.path)
523
+
524
+ def _segment(self, name: str, args: dict) -> str:
525
+ kept = self.value({name: args.get(name, "")})[name]
526
+ return REDACTED if kept == REDACTED else urllib.parse.quote(str(kept), safe="")
@@ -4,11 +4,11 @@ The TypeScript twin is ``worldData/cast.ts``; ``tests/conformance/csv_cast.json`
4
4
  them to the same answers. ``integer`` and ``number`` parse as numbers (a number is
5
5
  always a float), ``boolean`` takes true/false/1/0 in any case, ``array`` and
6
6
  ``object`` parse as JSON of that shape, ``string`` is the text as given. An empty
7
- cell is ``None`` when the field's type admits null, left out when the field is
8
- optional, the empty string for a required string, and refused for any other
9
- required field. A union type (``integer | string``) tries each member in order,
10
- ``string`` last. A cell that fits none is refused, naming the field and the cell;
11
- a column the schema does not name stays text (the gate refuses it, as it always did).
7
+ cell is ``None`` when the field's type admits null, the empty string for a required
8
+ string, and left out otherwise. A union type (``integer | string``) tries each member
9
+ in order, ``string`` last. A cell that fits none stays its text, as does a column the
10
+ schema does not name: the contract's check then refuses the row, naming its entity,
11
+ key and field, as it refuses a JSON row, so a file reports every bad row at once.
12
12
 
13
13
  Standard library only, so the bridge and the api bundle can load it.
14
14
  """
@@ -23,8 +23,6 @@ from dataclasses import dataclass
23
23
  from pathlib import Path
24
24
  from typing import Any
25
25
 
26
- from .validation import fail
27
-
28
26
  INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
29
27
  NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
30
28
  BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
@@ -51,21 +49,6 @@ class FieldType:
51
49
  return described
52
50
 
53
51
 
54
- def _article(kind: str) -> str:
55
- return "an" if kind[:1].lower() in "aeiou" else "a"
56
-
57
-
58
- def refuse(where: str, field: str, cell: str | None, kind: str) -> None:
59
- """The row-level refusal a cell that will not cast earns (same words as the TypeScript twin)."""
60
- if cell is None:
61
- fail("rows", f"{where}: {field} is empty; the field is a required {kind}", where)
62
- fail(
63
- "rows",
64
- f"{where}: {field} = {json.dumps(cell)} is not {_article(kind)} {kind}",
65
- where,
66
- )
67
-
68
-
69
52
  def type_members(kind: str | None) -> list[str]:
70
53
  """The typed members of a field's ``type`` (``"integer | null"`` -> integer, null); an
71
54
  unknown word (a ``$ref`` name) is dropped."""
@@ -123,10 +106,11 @@ class FieldCaster:
123
106
  self.required = required
124
107
  self.members = type_members(kind)
125
108
 
126
- def cast(self, cell: str | None, where: str) -> Any:
127
- """The typed value; :data:`OMIT` to leave the field out of the row."""
109
+ def cast(self, cell: str | None) -> Any:
110
+ """The typed value; :data:`OMIT` to leave the field out of the row; the cell as given when
111
+ it fits no member, for the contract's check to refuse."""
128
112
  if cell is None:
129
- return self._empty(where)
113
+ return self._empty()
130
114
  if not self.members:
131
115
  return cell
132
116
  text = cell.strip()
@@ -134,18 +118,14 @@ class FieldCaster:
134
118
  value = _CAST.get(member, _no_cast)(text)
135
119
  if value is not _MISSING:
136
120
  return value
137
- if "string" in self.members:
138
- return cell
139
- refuse(where, self.name, cell, self.type or "value")
121
+ return cell
140
122
 
141
- def _empty(self, where: str) -> Any:
123
+ def _empty(self) -> Any:
142
124
  if not self.members or "null" in self.members:
143
125
  return None
144
- if not self.required:
145
- return OMIT
146
- if "string" in self.members:
126
+ if self.required and "string" in self.members:
147
127
  return ""
148
- refuse(where, self.name, None, self.type or "value")
128
+ return OMIT
149
129
 
150
130
 
151
131
  #: What :meth:`FieldCaster.cast` answers for an empty optional cell: the field stays out of the row.
@@ -158,11 +138,11 @@ class RowCaster:
158
138
  def __init__(self, fields: Iterable[FieldType]):
159
139
  self._fields = {f.name: FieldCaster(f.name, f.type, f.required) for f in fields}
160
140
 
161
- def row(self, record: dict[str, str | None], where: str) -> dict:
141
+ def row(self, record: dict[str, str | None]) -> dict:
162
142
  row: dict = {}
163
143
  for name, cell in record.items():
164
144
  caster = self._fields.get(name)
165
- value = cell if caster is None else caster.cast(cell, where)
145
+ value = cell if caster is None else caster.cast(cell)
166
146
  if value is not OMIT:
167
147
  row[name] = value
168
148
  return row