@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -10,10 +10,11 @@ connection is described once::
10
10
  base_url = "https://api.spycloud.io/investigations-v2"
11
11
 
12
12
  [auth]
13
- scheme = "header" # header | bearer | basic | query | none
13
+ scheme = "header" # header | bearer | basic | query | oauth2 | none
14
14
  name = "x-api-key" # header or query parameter that carries the secret
15
15
  secret_env = "SPYCLOUD_API_KEY"
16
16
  headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" } # more credential headers, each named by an env var
17
+ # [auth.login] and [auth.downloads]: the vendor's own login and signed links (door.py)
17
18
 
18
19
  [capture]
19
20
  requests_per_minute = 30 # pacing between requests; providers ask us to be sparing
@@ -58,7 +59,6 @@ the standard library, so the credential-free bridge can load it.
58
59
 
59
60
  from __future__ import annotations
60
61
 
61
- import base64
62
62
  import hashlib
63
63
  import json
64
64
  import os
@@ -76,23 +76,34 @@ from typing import Any, Callable, Iterable
76
76
 
77
77
  from .compiler import check_bundle, compile_bundle
78
78
  from .database import WorldDatabase
79
+ from .door import VendorAuth
79
80
  from .primary_key import PrimaryKey
80
81
  from .transforms import apply_transforms
81
- from .validation import FIELD_NAME, ContractError, fail, operation_name, parse_toml
82
+ from .validation import (
83
+ FIELD_NAME,
84
+ METHODS,
85
+ PLACEHOLDER,
86
+ ContractError,
87
+ fail,
88
+ names_of,
89
+ operation_name,
90
+ parse_toml,
91
+ path_pattern,
92
+ route_path,
93
+ string_of,
94
+ table_of,
95
+ )
96
+ from .webhooks import Webhooks
82
97
 
83
98
  SPEC = "connector.toml"
84
99
  CAPTURES = "captures"
85
100
  MANIFEST = "manifest.json"
86
- SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
87
- DEFAULT_TOKEN_PATH = "/oauth/token"
88
- METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
89
101
  SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
90
102
  #: A declared query parameter as the vendor names it: dotted (``query.cond``,
91
103
  #: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
92
104
  #: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
93
105
  #: match or filter on it.
94
106
  PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
95
- PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
96
107
  # Pairs, not a dict, so a repeated key survives.
97
108
  Params = list[tuple[str, str]]
98
109
  MAX_BODY = 8 * 1024 * 1024
@@ -139,18 +150,21 @@ class Operation:
139
150
  def placeholders(self) -> tuple[str, ...]:
140
151
  return tuple(PLACEHOLDER.findall(self.path))
141
152
 
153
+ @property
154
+ def pattern(self) -> re.Pattern:
155
+ """The route's path as a pattern over a request's path, each ``{arg}`` one segment."""
156
+ return path_pattern(self.path)
157
+
158
+ def path_args(self, path: str) -> dict[str, str]:
159
+ found = self.pattern.match(path)
160
+ return {k: urllib.parse.unquote(v) for k, v in found.groupdict().items()} if found else {}
161
+
142
162
 
143
163
  @dataclass(frozen=True)
144
164
  class Spec:
145
165
  slug: str
146
166
  base_url: str
147
- scheme: str
148
- auth_name: str | None
149
- secret_env: str | None
150
- extra_headers: dict[str, str]
151
- token_url: str | None
152
- client_id_env: str | None
153
- scope: str | None
167
+ auth: VendorAuth
154
168
  requests_per_minute: int
155
169
  timeout_seconds: float
156
170
  max_pages: int
@@ -162,6 +176,12 @@ class Spec:
162
176
  round_fields: dict[str, int] = field(default_factory=dict)
163
177
  preserve_fields: frozenset[str] = frozenset()
164
178
  operations: dict[str, Operation] = field(default_factory=dict)
179
+ webhooks: Webhooks | None = None
180
+
181
+ @property
182
+ def scheme(self) -> str:
183
+ """The ``[auth] scheme`` the connection declares."""
184
+ return self.auth.scheme.name
165
185
 
166
186
  def operation(self, name: str) -> Operation:
167
187
  if name not in self.operations:
@@ -190,23 +210,8 @@ class Spec:
190
210
  "`gateway worlds schema entity add`, or point the route at one that exists",
191
211
  location,
192
212
  )
193
-
194
-
195
- def _table(value: Any, location: str) -> dict:
196
- if not isinstance(value, dict):
197
- fail("spec", "Expected a table", location)
198
- return value
199
-
200
-
201
- def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
202
- if key not in table:
203
- if required:
204
- fail("spec", f"Missing {key}", location)
205
- return default
206
- value = table[key]
207
- if not isinstance(value, str) or not value.strip():
208
- fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
209
- return value.strip()
213
+ if self.webhooks is not None:
214
+ self.webhooks.check_entity(known)
210
215
 
211
216
 
212
217
  RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
@@ -216,7 +221,7 @@ def parse_redact(value: Any, location: str) -> RedactTable:
216
221
  """A ``[redact]`` table as ``(hash, drop, round, preserve)`` — the one grammar for
217
222
  the policy wherever it is declared (``connector.toml`` for a vendor world,
218
223
  ``gateway-env.toml`` for a tools-only schema world)."""
219
- redact = _table(value, location)
224
+ redact = table_of(value, location)
220
225
  hash_fields = frozenset(_paths(redact, "hash", location))
221
226
  drop_fields = frozenset(_paths(redact, "drop", location))
222
227
  rounding = redact.get("round", {})
@@ -260,15 +265,6 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
260
265
  return verbs
261
266
 
262
267
 
263
- def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
264
- value = table.get(key, [])
265
- if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
266
- fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
267
- if len(set(value)) != len(value):
268
- fail("spec", f"{key} repeats a name", f"{location}/{key}")
269
- return tuple(value)
270
-
271
-
272
268
  def pairs_of(params: Any) -> Params:
273
269
  """Query parameters as an ordered list of ``(name, value)`` pairs.
274
270
 
@@ -315,81 +311,68 @@ def parse_spec(text: str) -> Spec:
315
311
 
316
312
  def spec_from(raw: dict) -> Spec:
317
313
  """The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
318
- connector = _table(raw.get("connector"), "/connector")
319
- slug = _string(connector, "slug", "/connector")
314
+ connector = table_of(raw.get("connector"), "/connector")
315
+ slug = string_of(connector, "slug", "/connector")
320
316
  if not SLUG.match(slug):
321
317
  fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
322
- base_url = _string(connector, "base_url", "/connector")
318
+ base_url = string_of(connector, "base_url", "/connector")
323
319
  parsed = urllib.parse.urlsplit(base_url)
324
320
  if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
325
321
  fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
326
322
 
327
- auth = _table(raw.get("auth", {"scheme": "none"}), "/auth")
328
- scheme = _string(auth, "scheme", "/auth")
329
- if scheme not in SCHEMES:
330
- fail("spec", f"auth.scheme must be one of {', '.join(SCHEMES)}", "/auth/scheme")
331
- auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
332
- secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
333
- if scheme == "none" and (auth_name or secret_env):
334
- fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
335
- token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
336
- client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
337
- scope = _string(auth, "scope", "/auth", required=False)
338
- if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
339
- fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
340
- if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
341
- fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
342
- extra_headers = auth.get("headers", {})
343
- if not isinstance(extra_headers, dict) or not all(
344
- isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
345
- ):
346
- fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
347
- if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
348
- fail("spec", "auth.headers repeats the secret header", "/auth/headers")
349
-
350
- capture = _table(raw.get("capture", {}), "/capture")
351
- cursor_param = _string(capture, "cursor_param", "/capture", required=False)
352
- cursor_field = _string(capture, "cursor_field", "/capture", required=False)
353
- cursor_header = _string(capture, "cursor_header", "/capture", required=False)
323
+ capture = table_of(raw.get("capture", {}), "/capture")
324
+ cursor_param = string_of(capture, "cursor_param", "/capture", required=False)
325
+ cursor_field = string_of(capture, "cursor_field", "/capture", required=False)
326
+ cursor_header = string_of(capture, "cursor_header", "/capture", required=False)
354
327
  if cursor_field and cursor_header:
355
328
  fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
356
329
  if bool(cursor_param) != bool(cursor_field or cursor_header):
357
330
  fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
358
331
 
359
332
  hash_fields, drop_fields, round_fields, preserve_fields = parse_redact(raw.get("redact", {}), "/redact")
333
+ operations = _operations(raw.get("operations", []))
334
+
335
+ return Spec(
336
+ slug=slug,
337
+ base_url=base_url.rstrip("/"),
338
+ auth=VendorAuth.parse(raw.get("auth", {"scheme": "none"}), operations),
339
+ requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
340
+ timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
341
+ max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
342
+ cursor_param=cursor_param,
343
+ cursor_field=cursor_field,
344
+ cursor_header=cursor_header,
345
+ hash_fields=hash_fields,
346
+ drop_fields=drop_fields,
347
+ round_fields=round_fields,
348
+ preserve_fields=preserve_fields,
349
+ operations=operations,
350
+ webhooks=Webhooks.parse(raw.get("webhooks")),
351
+ )
360
352
 
353
+
354
+ def _operations(listed: Any) -> dict[str, Operation]:
355
+ """Every ``[[operations]]`` entry by name; two that would answer the same request are refused."""
361
356
  operations: dict[str, Operation] = {}
362
357
  routes: dict[tuple[str, str, tuple[str, ...]], str] = {}
363
- listed = raw.get("operations", [])
364
358
  if not isinstance(listed, list) or not listed:
365
359
  fail("spec", "Declare at least one [[operations]] entry", "/operations")
366
360
  for index, entry in enumerate(listed):
367
361
  location = f"/operations/{index}"
368
- table = _table(entry, location)
369
- name = _string(table, "name", location)
362
+ table = table_of(entry, location)
363
+ name = string_of(table, "name", location)
370
364
  operation_name(name, f"{location}/name")
371
365
  if name in operations:
372
366
  fail("spec", f"operation {name} is declared twice", f"{location}/name")
373
367
  methods = _verbs(table, location)
374
- method = _string(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
368
+ method = string_of(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
375
369
  if method not in METHODS:
376
370
  fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
377
371
  if methods and method not in methods:
378
372
  fail("spec", f"method {method} must be one of methods ({', '.join(methods)})", f"{location}/method")
379
- path = _string(table, "path", location)
380
- if not path.startswith("/") or "?" in path or "#" in path:
381
- fail("spec", "path must start with / and carry no query", f"{location}/path")
382
- placeholders = PLACEHOLDER.findall(path)
383
- repeated = [p for p in placeholders if placeholders.count(p) > 1]
384
- if repeated:
385
- fail(
386
- "spec",
387
- f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
388
- f"{location}/path",
389
- argument=repeated[0],
390
- )
391
- params = _names(table, "params", location, PARAM)
392
- entity = _string(table, "entity", location, required=False)
373
+ path = route_path(string_of(table, "path", location) or "", f"{location}/path")
374
+ params = names_of(table, "params", location, PARAM)
375
+ entity = string_of(table, "entity", location, required=False)
393
376
  results = table.get("results", "")
394
377
  if not isinstance(results, str):
395
378
  fail("spec", "results must be a dotted path string", f"{location}/results")
@@ -416,29 +399,7 @@ def spec_from(raw: dict) -> Spec:
416
399
  f"{location}/path",
417
400
  )
418
401
  routes[route] = name
419
-
420
- return Spec(
421
- slug=slug,
422
- base_url=base_url.rstrip("/"),
423
- scheme=scheme,
424
- auth_name=auth_name,
425
- secret_env=secret_env,
426
- extra_headers=dict(extra_headers),
427
- token_url=token_url if scheme == "oauth2" else None,
428
- client_id_env=client_id_env if scheme == "oauth2" else None,
429
- scope=scope if scheme == "oauth2" else None,
430
- requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
431
- timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
432
- max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
433
- cursor_param=cursor_param,
434
- cursor_field=cursor_field,
435
- cursor_header=cursor_header,
436
- hash_fields=hash_fields,
437
- drop_fields=drop_fields,
438
- round_fields=round_fields,
439
- preserve_fields=preserve_fields,
440
- operations=operations,
441
- )
402
+ return operations
442
403
 
443
404
 
444
405
  def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
@@ -451,8 +412,8 @@ def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
451
412
 
452
413
 
453
414
  def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...], dict[str, str]]:
454
- table = _table(raw, location)
455
- drop = _names(table, "drop", location)
415
+ table = table_of(raw, location)
416
+ drop = names_of(table, "drop", location)
456
417
  carry_args = _mapping(table, "carry_args", location)
457
418
  explode: list[Explode] = []
458
419
  listed = table.get("explode", [])
@@ -460,9 +421,9 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
460
421
  fail("spec", "explode must be a list of tables", f"{location}/explode")
461
422
  for index, entry in enumerate(listed):
462
423
  where = f"{location}/explode/{index}"
463
- item = _table(entry, where)
464
- field_name = _string(item, "field", where)
465
- entity = _string(item, "entity", where)
424
+ item = table_of(entry, where)
425
+ field_name = string_of(item, "field", where)
426
+ entity = string_of(item, "entity", where)
466
427
  carry = _mapping(item, "carry", where)
467
428
  take = _mapping(item, "take", where)
468
429
  if not carry and not take:
@@ -501,20 +462,18 @@ def load_spec(world: str | Path) -> Spec:
501
462
  def _secret_value(spec: Spec, secret: str | None) -> str:
502
463
  if secret is not None:
503
464
  return secret
504
- value = os.environ.get(spec.secret_env or "")
465
+ value = os.environ.get(spec.auth.secret_env or "")
505
466
  if not value:
506
- fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
467
+ fail("secret", f"Set {spec.auth.secret_env} to the connection's credential", "/auth/secret_env")
507
468
  return value
508
469
 
509
470
 
510
471
  def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
511
- """The value the requests carry: the credential itself, or for oauth2 the
512
- access token minted from it."""
513
- if spec.scheme == "none":
514
- return None
515
- if spec.scheme == "oauth2":
516
- return mint_token(spec, secret=secret, opener=opener)
517
- return _secret_value(spec, secret)
472
+ """The value the requests carry: the credential itself, or the token the
473
+ scheme exchanges it for first (oauth2)."""
474
+ return spec.auth.scheme.capture_value(
475
+ lambda: _secret_value(spec, secret), lambda: mint_token(spec, secret=secret, opener=opener)
476
+ )
518
477
 
519
478
 
520
479
  def build_request(
@@ -540,19 +499,14 @@ def build_request(
540
499
  query = list(pairs_of(params))
541
500
  recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
542
501
  headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
543
- for header, variable in spec.extra_headers.items():
502
+ for header, variable in spec.auth.extra_headers.items():
544
503
  value = os.environ.get(variable)
545
504
  if not value:
546
505
  fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
547
506
  headers[header] = value
548
- if spec.scheme == "header":
549
- headers[spec.auth_name or ""] = secret or ""
550
- elif spec.scheme in ("bearer", "oauth2"):
551
- headers["Authorization"] = f"Bearer {secret}"
552
- elif spec.scheme == "basic":
553
- headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
554
- elif spec.scheme == "query":
555
- query.append((spec.auth_name or "", secret or ""))
507
+ carried_headers, carried_query = spec.auth.scheme.carry(secret or "")
508
+ headers.update(carried_headers)
509
+ query.extend(carried_query.items())
556
510
  url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
557
511
  return urllib.request.Request(url, method=operation.method, headers=headers), recorded
558
512
 
@@ -567,29 +521,15 @@ def undeclared_params(spec: Spec, operation: Operation, params: Params) -> list[
567
521
  return seen
568
522
 
569
523
 
570
- def token_endpoint(spec: Spec) -> str:
571
- """Where an oauth2 connection exchanges its client credentials."""
572
- url = spec.token_url or DEFAULT_TOKEN_PATH
573
- return url if url.startswith(("https://", "http://")) else spec.base_url + url
574
-
575
-
576
524
  def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
577
525
  """Exchange the client credentials for an access token (RFC 6749 §4.4).
578
526
 
579
527
  The client id and secret come from the environment the way every other
580
528
  credential does; the token is what the captures then send as a bearer.
581
529
  Nothing about the exchange is recorded, so the secret never reaches disk."""
582
- if spec.scheme != "oauth2":
583
- fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
584
- client_id = os.environ.get(spec.client_id_env or "")
585
- if not client_id:
586
- fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
587
- client_secret = _secret_value(spec, secret)
588
- form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
589
- if spec.scope:
590
- form["scope"] = spec.scope
530
+ endpoint, form = spec.auth.scheme.token_request(spec.base_url, lambda: _secret_value(spec, secret), os.environ)
591
531
  request = urllib.request.Request(
592
- token_endpoint(spec),
532
+ endpoint,
593
533
  method="POST",
594
534
  data=urllib.parse.urlencode(form).encode(),
595
535
  headers={
@@ -808,7 +748,9 @@ def capture(
808
748
  following = urllib.parse.urlsplit(cursor)
809
749
  if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
810
750
  fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
811
- page_params = [(k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != (spec.auth_name or "")]
751
+ page_params = [
752
+ (k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != spec.auth.scheme.secret_param
753
+ ]
812
754
  elif cursor:
813
755
  cursor_param = spec.cursor_param or ""
814
756
  page_params = [(k, v) for k, v in page_params if k != cursor_param] + [(cursor_param, cursor)]
@@ -20,6 +20,7 @@ base_url = "https://your-domain.atlassian.net"
20
20
  # UI's; a world served with no key accepts any non-empty one.
21
21
  [auth]
22
22
  scheme = "basic"
23
+ credential = "password"
23
24
  secret_env = "CONFLUENCE_API_TOKEN"
24
25
 
25
26
  [capture]
@@ -60,8 +61,6 @@ entry = "/"
60
61
  api_path = "/__world-api"
61
62
  generated = false
62
63
 
63
- # ---------------------------------------------------------------- spaces
64
-
65
64
  [[operations]]
66
65
  name = "get_spaces"
67
66
  method = "GET"
@@ -96,8 +95,6 @@ filter = { id = "spaceId", status = "status", title = "title" }
96
95
  match = { title = "contains" }
97
96
  sample = { id = "65537" }
98
97
 
99
- # ---------------------------------------------------------------- pages
100
-
101
98
  [[operations]]
102
99
  name = "get_pages"
103
100
  method = "GET"
@@ -193,8 +190,6 @@ results = "results"
193
190
  filter = { id = "pageId" }
194
191
  sample = { id = "100063" }
195
192
 
196
- # ---------------------------------------------------------------- comments
197
-
198
193
  [[operations]]
199
194
  name = "get_page_footer_comments"
200
195
  method = "GET"
@@ -265,8 +260,6 @@ handler = "update_inline_comment"
265
260
  sample = { commentId = "cmt-005" }
266
261
  sample_body = { version = { number = 2 }, resolved = true }
267
262
 
268
- # ---------------------------------------------------------------- labels
269
-
270
263
  [[operations]]
271
264
  name = "get_labels"
272
265
  method = "GET"
@@ -289,8 +282,6 @@ action = "handler"
289
282
  handler = "get_page_labels"
290
283
  sample = { id = "100063" }
291
284
 
292
- # ---------------------------------------------------------------- the ui's own reads
293
- #
294
285
  # One route per call in the UI's api client. Every one is a handler: these are
295
286
  # composed read models (a page with its ancestors, children, comments and versions;
296
287
  # a space tree), not vendor routes, and no vendor document describes them.
@@ -417,8 +408,6 @@ results = ""
417
408
  action = "handler"
418
409
  handler = "ui_labels"
419
410
 
420
- # ---------------------------------------------------------------- the ui's writes
421
-
422
411
  [[operations]]
423
412
  name = "ui_add_comment"
424
413
  method = "POST"
@@ -33,7 +33,8 @@ hosted call into such a handler answers 400 {error: "handler", detail, handler}:
33
33
  Rows per entity, as {entity: count}.
34
34
 
35
35
  A handler that raises AttributeError, TypeError or NameError is answered 400 with
36
- that structured body; any other exception is answered 500 {error: "engine", ...}.
36
+ that structured body; `raise Refusal(404, body)` (from gateway_env) answers that 4xx
37
+ and body as the vendor's refusal; any other exception is answered 500 {error: "engine", ...}.
37
38
  `schema init --connector custom --with-example-tool` writes the example in full.
38
39
  """
39
40