@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -10,10 +10,11 @@ connection is described once::
10
10
  base_url = "https://api.spycloud.io/investigations-v2"
11
11
 
12
12
  [auth]
13
- scheme = "header" # header | bearer | basic | query | none
13
+ scheme = "header" # header | bearer | basic | query | oauth2 | none
14
14
  name = "x-api-key" # header or query parameter that carries the secret
15
15
  secret_env = "SPYCLOUD_API_KEY"
16
16
  headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" } # more credential headers, each named by an env var
17
+ # [auth.login] and [auth.downloads]: the vendor's own login and signed links (door.py)
17
18
 
18
19
  [capture]
19
20
  requests_per_minute = 30 # pacing between requests; providers ask us to be sparing
@@ -58,7 +59,6 @@ the standard library, so the credential-free bridge can load it.
58
59
 
59
60
  from __future__ import annotations
60
61
 
61
- import base64
62
62
  import hashlib
63
63
  import json
64
64
  import os
@@ -76,23 +76,33 @@ from typing import Any, Callable, Iterable
76
76
 
77
77
  from .compiler import check_bundle, compile_bundle
78
78
  from .database import WorldDatabase
79
+ from .door import VendorAuth
79
80
  from .primary_key import PrimaryKey
80
81
  from .transforms import apply_transforms
81
- from .validation import FIELD_NAME, ContractError, fail, operation_name, parse_toml
82
+ from .validation import (
83
+ FIELD_NAME,
84
+ METHODS,
85
+ PLACEHOLDER,
86
+ ContractError,
87
+ fail,
88
+ names_of,
89
+ operation_name,
90
+ parse_toml,
91
+ path_pattern,
92
+ route_path,
93
+ string_of,
94
+ table_of,
95
+ )
82
96
 
83
97
  SPEC = "connector.toml"
84
98
  CAPTURES = "captures"
85
99
  MANIFEST = "manifest.json"
86
- SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
87
- DEFAULT_TOKEN_PATH = "/oauth/token"
88
- METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
89
100
  SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
90
101
  #: A declared query parameter as the vendor names it: dotted (``query.cond``,
91
102
  #: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
92
103
  #: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
93
104
  #: match or filter on it.
94
105
  PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
95
- PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
96
106
  # Pairs, not a dict, so a repeated key survives.
97
107
  Params = list[tuple[str, str]]
98
108
  MAX_BODY = 8 * 1024 * 1024
@@ -139,18 +149,21 @@ class Operation:
139
149
  def placeholders(self) -> tuple[str, ...]:
140
150
  return tuple(PLACEHOLDER.findall(self.path))
141
151
 
152
+ @property
153
+ def pattern(self) -> re.Pattern:
154
+ """The route's path as a pattern over a request's path, each ``{arg}`` one segment."""
155
+ return path_pattern(self.path)
156
+
157
+ def path_args(self, path: str) -> dict[str, str]:
158
+ found = self.pattern.match(path)
159
+ return {k: urllib.parse.unquote(v) for k, v in found.groupdict().items()} if found else {}
160
+
142
161
 
143
162
  @dataclass(frozen=True)
144
163
  class Spec:
145
164
  slug: str
146
165
  base_url: str
147
- scheme: str
148
- auth_name: str | None
149
- secret_env: str | None
150
- extra_headers: dict[str, str]
151
- token_url: str | None
152
- client_id_env: str | None
153
- scope: str | None
166
+ auth: VendorAuth
154
167
  requests_per_minute: int
155
168
  timeout_seconds: float
156
169
  max_pages: int
@@ -163,6 +176,11 @@ class Spec:
163
176
  preserve_fields: frozenset[str] = frozenset()
164
177
  operations: dict[str, Operation] = field(default_factory=dict)
165
178
 
179
+ @property
180
+ def scheme(self) -> str:
181
+ """The ``[auth] scheme`` the connection declares."""
182
+ return self.auth.scheme.name
183
+
166
184
  def operation(self, name: str) -> Operation:
167
185
  if name not in self.operations:
168
186
  fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
@@ -192,23 +210,6 @@ class Spec:
192
210
  )
193
211
 
194
212
 
195
- def _table(value: Any, location: str) -> dict:
196
- if not isinstance(value, dict):
197
- fail("spec", "Expected a table", location)
198
- return value
199
-
200
-
201
- def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
202
- if key not in table:
203
- if required:
204
- fail("spec", f"Missing {key}", location)
205
- return default
206
- value = table[key]
207
- if not isinstance(value, str) or not value.strip():
208
- fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
209
- return value.strip()
210
-
211
-
212
213
  RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
213
214
 
214
215
 
@@ -216,7 +217,7 @@ def parse_redact(value: Any, location: str) -> RedactTable:
216
217
  """A ``[redact]`` table as ``(hash, drop, round, preserve)`` — the one grammar for
217
218
  the policy wherever it is declared (``connector.toml`` for a vendor world,
218
219
  ``gateway-env.toml`` for a tools-only schema world)."""
219
- redact = _table(value, location)
220
+ redact = table_of(value, location)
220
221
  hash_fields = frozenset(_paths(redact, "hash", location))
221
222
  drop_fields = frozenset(_paths(redact, "drop", location))
222
223
  rounding = redact.get("round", {})
@@ -260,15 +261,6 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
260
261
  return verbs
261
262
 
262
263
 
263
- def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
264
- value = table.get(key, [])
265
- if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
266
- fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
267
- if len(set(value)) != len(value):
268
- fail("spec", f"{key} repeats a name", f"{location}/{key}")
269
- return tuple(value)
270
-
271
-
272
264
  def pairs_of(params: Any) -> Params:
273
265
  """Query parameters as an ordered list of ``(name, value)`` pairs.
274
266
 
@@ -315,81 +307,67 @@ def parse_spec(text: str) -> Spec:
315
307
 
316
308
  def spec_from(raw: dict) -> Spec:
317
309
  """The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
318
- connector = _table(raw.get("connector"), "/connector")
319
- slug = _string(connector, "slug", "/connector")
310
+ connector = table_of(raw.get("connector"), "/connector")
311
+ slug = string_of(connector, "slug", "/connector")
320
312
  if not SLUG.match(slug):
321
313
  fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
322
- base_url = _string(connector, "base_url", "/connector")
314
+ base_url = string_of(connector, "base_url", "/connector")
323
315
  parsed = urllib.parse.urlsplit(base_url)
324
316
  if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
325
317
  fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
326
318
 
327
- auth = _table(raw.get("auth", {"scheme": "none"}), "/auth")
328
- scheme = _string(auth, "scheme", "/auth")
329
- if scheme not in SCHEMES:
330
- fail("spec", f"auth.scheme must be one of {', '.join(SCHEMES)}", "/auth/scheme")
331
- auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
332
- secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
333
- if scheme == "none" and (auth_name or secret_env):
334
- fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
335
- token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
336
- client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
337
- scope = _string(auth, "scope", "/auth", required=False)
338
- if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
339
- fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
340
- if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
341
- fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
342
- extra_headers = auth.get("headers", {})
343
- if not isinstance(extra_headers, dict) or not all(
344
- isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
345
- ):
346
- fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
347
- if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
348
- fail("spec", "auth.headers repeats the secret header", "/auth/headers")
349
-
350
- capture = _table(raw.get("capture", {}), "/capture")
351
- cursor_param = _string(capture, "cursor_param", "/capture", required=False)
352
- cursor_field = _string(capture, "cursor_field", "/capture", required=False)
353
- cursor_header = _string(capture, "cursor_header", "/capture", required=False)
319
+ capture = table_of(raw.get("capture", {}), "/capture")
320
+ cursor_param = string_of(capture, "cursor_param", "/capture", required=False)
321
+ cursor_field = string_of(capture, "cursor_field", "/capture", required=False)
322
+ cursor_header = string_of(capture, "cursor_header", "/capture", required=False)
354
323
  if cursor_field and cursor_header:
355
324
  fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
356
325
  if bool(cursor_param) != bool(cursor_field or cursor_header):
357
326
  fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
358
327
 
359
328
  hash_fields, drop_fields, round_fields, preserve_fields = parse_redact(raw.get("redact", {}), "/redact")
329
+ operations = _operations(raw.get("operations", []))
330
+
331
+ return Spec(
332
+ slug=slug,
333
+ base_url=base_url.rstrip("/"),
334
+ auth=VendorAuth.parse(raw.get("auth", {"scheme": "none"}), operations),
335
+ requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
336
+ timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
337
+ max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
338
+ cursor_param=cursor_param,
339
+ cursor_field=cursor_field,
340
+ cursor_header=cursor_header,
341
+ hash_fields=hash_fields,
342
+ drop_fields=drop_fields,
343
+ round_fields=round_fields,
344
+ preserve_fields=preserve_fields,
345
+ operations=operations,
346
+ )
360
347
 
348
+
349
+ def _operations(listed: Any) -> dict[str, Operation]:
350
+ """Every ``[[operations]]`` entry by name; two that would answer the same request are refused."""
361
351
  operations: dict[str, Operation] = {}
362
352
  routes: dict[tuple[str, str, tuple[str, ...]], str] = {}
363
- listed = raw.get("operations", [])
364
353
  if not isinstance(listed, list) or not listed:
365
354
  fail("spec", "Declare at least one [[operations]] entry", "/operations")
366
355
  for index, entry in enumerate(listed):
367
356
  location = f"/operations/{index}"
368
- table = _table(entry, location)
369
- name = _string(table, "name", location)
357
+ table = table_of(entry, location)
358
+ name = string_of(table, "name", location)
370
359
  operation_name(name, f"{location}/name")
371
360
  if name in operations:
372
361
  fail("spec", f"operation {name} is declared twice", f"{location}/name")
373
362
  methods = _verbs(table, location)
374
- method = _string(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
363
+ method = string_of(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
375
364
  if method not in METHODS:
376
365
  fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
377
366
  if methods and method not in methods:
378
367
  fail("spec", f"method {method} must be one of methods ({', '.join(methods)})", f"{location}/method")
379
- path = _string(table, "path", location)
380
- if not path.startswith("/") or "?" in path or "#" in path:
381
- fail("spec", "path must start with / and carry no query", f"{location}/path")
382
- placeholders = PLACEHOLDER.findall(path)
383
- repeated = [p for p in placeholders if placeholders.count(p) > 1]
384
- if repeated:
385
- fail(
386
- "spec",
387
- f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
388
- f"{location}/path",
389
- argument=repeated[0],
390
- )
391
- params = _names(table, "params", location, PARAM)
392
- entity = _string(table, "entity", location, required=False)
368
+ path = route_path(string_of(table, "path", location) or "", f"{location}/path")
369
+ params = names_of(table, "params", location, PARAM)
370
+ entity = string_of(table, "entity", location, required=False)
393
371
  results = table.get("results", "")
394
372
  if not isinstance(results, str):
395
373
  fail("spec", "results must be a dotted path string", f"{location}/results")
@@ -416,29 +394,7 @@ def spec_from(raw: dict) -> Spec:
416
394
  f"{location}/path",
417
395
  )
418
396
  routes[route] = name
419
-
420
- return Spec(
421
- slug=slug,
422
- base_url=base_url.rstrip("/"),
423
- scheme=scheme,
424
- auth_name=auth_name,
425
- secret_env=secret_env,
426
- extra_headers=dict(extra_headers),
427
- token_url=token_url if scheme == "oauth2" else None,
428
- client_id_env=client_id_env if scheme == "oauth2" else None,
429
- scope=scope if scheme == "oauth2" else None,
430
- requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
431
- timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
432
- max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
433
- cursor_param=cursor_param,
434
- cursor_field=cursor_field,
435
- cursor_header=cursor_header,
436
- hash_fields=hash_fields,
437
- drop_fields=drop_fields,
438
- round_fields=round_fields,
439
- preserve_fields=preserve_fields,
440
- operations=operations,
441
- )
397
+ return operations
442
398
 
443
399
 
444
400
  def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
@@ -451,8 +407,8 @@ def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
451
407
 
452
408
 
453
409
  def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...], dict[str, str]]:
454
- table = _table(raw, location)
455
- drop = _names(table, "drop", location)
410
+ table = table_of(raw, location)
411
+ drop = names_of(table, "drop", location)
456
412
  carry_args = _mapping(table, "carry_args", location)
457
413
  explode: list[Explode] = []
458
414
  listed = table.get("explode", [])
@@ -460,9 +416,9 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
460
416
  fail("spec", "explode must be a list of tables", f"{location}/explode")
461
417
  for index, entry in enumerate(listed):
462
418
  where = f"{location}/explode/{index}"
463
- item = _table(entry, where)
464
- field_name = _string(item, "field", where)
465
- entity = _string(item, "entity", where)
419
+ item = table_of(entry, where)
420
+ field_name = string_of(item, "field", where)
421
+ entity = string_of(item, "entity", where)
466
422
  carry = _mapping(item, "carry", where)
467
423
  take = _mapping(item, "take", where)
468
424
  if not carry and not take:
@@ -501,20 +457,18 @@ def load_spec(world: str | Path) -> Spec:
501
457
  def _secret_value(spec: Spec, secret: str | None) -> str:
502
458
  if secret is not None:
503
459
  return secret
504
- value = os.environ.get(spec.secret_env or "")
460
+ value = os.environ.get(spec.auth.secret_env or "")
505
461
  if not value:
506
- fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
462
+ fail("secret", f"Set {spec.auth.secret_env} to the connection's credential", "/auth/secret_env")
507
463
  return value
508
464
 
509
465
 
510
466
  def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
511
- """The value the requests carry: the credential itself, or for oauth2 the
512
- access token minted from it."""
513
- if spec.scheme == "none":
514
- return None
515
- if spec.scheme == "oauth2":
516
- return mint_token(spec, secret=secret, opener=opener)
517
- return _secret_value(spec, secret)
467
+ """The value the requests carry: the credential itself, or the token the
468
+ scheme exchanges it for first (oauth2)."""
469
+ return spec.auth.scheme.capture_value(
470
+ lambda: _secret_value(spec, secret), lambda: mint_token(spec, secret=secret, opener=opener)
471
+ )
518
472
 
519
473
 
520
474
  def build_request(
@@ -540,19 +494,14 @@ def build_request(
540
494
  query = list(pairs_of(params))
541
495
  recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
542
496
  headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
543
- for header, variable in spec.extra_headers.items():
497
+ for header, variable in spec.auth.extra_headers.items():
544
498
  value = os.environ.get(variable)
545
499
  if not value:
546
500
  fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
547
501
  headers[header] = value
548
- if spec.scheme == "header":
549
- headers[spec.auth_name or ""] = secret or ""
550
- elif spec.scheme in ("bearer", "oauth2"):
551
- headers["Authorization"] = f"Bearer {secret}"
552
- elif spec.scheme == "basic":
553
- headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
554
- elif spec.scheme == "query":
555
- query.append((spec.auth_name or "", secret or ""))
502
+ carried_headers, carried_query = spec.auth.scheme.carry(secret or "")
503
+ headers.update(carried_headers)
504
+ query.extend(carried_query.items())
556
505
  url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
557
506
  return urllib.request.Request(url, method=operation.method, headers=headers), recorded
558
507
 
@@ -567,29 +516,15 @@ def undeclared_params(spec: Spec, operation: Operation, params: Params) -> list[
567
516
  return seen
568
517
 
569
518
 
570
- def token_endpoint(spec: Spec) -> str:
571
- """Where an oauth2 connection exchanges its client credentials."""
572
- url = spec.token_url or DEFAULT_TOKEN_PATH
573
- return url if url.startswith(("https://", "http://")) else spec.base_url + url
574
-
575
-
576
519
  def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
577
520
  """Exchange the client credentials for an access token (RFC 6749 §4.4).
578
521
 
579
522
  The client id and secret come from the environment the way every other
580
523
  credential does; the token is what the captures then send as a bearer.
581
524
  Nothing about the exchange is recorded, so the secret never reaches disk."""
582
- if spec.scheme != "oauth2":
583
- fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
584
- client_id = os.environ.get(spec.client_id_env or "")
585
- if not client_id:
586
- fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
587
- client_secret = _secret_value(spec, secret)
588
- form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
589
- if spec.scope:
590
- form["scope"] = spec.scope
525
+ endpoint, form = spec.auth.scheme.token_request(spec.base_url, lambda: _secret_value(spec, secret), os.environ)
591
526
  request = urllib.request.Request(
592
- token_endpoint(spec),
527
+ endpoint,
593
528
  method="POST",
594
529
  data=urllib.parse.urlencode(form).encode(),
595
530
  headers={
@@ -808,7 +743,9 @@ def capture(
808
743
  following = urllib.parse.urlsplit(cursor)
809
744
  if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
810
745
  fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
811
- page_params = [(k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != (spec.auth_name or "")]
746
+ page_params = [
747
+ (k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != spec.auth.scheme.secret_param
748
+ ]
812
749
  elif cursor:
813
750
  cursor_param = spec.cursor_param or ""
814
751
  page_params = [(k, v) for k, v in page_params if k != cursor_param] + [(cursor_param, cursor)]
@@ -20,6 +20,7 @@ base_url = "https://your-domain.atlassian.net"
20
20
  # UI's; a world served with no key accepts any non-empty one.
21
21
  [auth]
22
22
  scheme = "basic"
23
+ credential = "password"
23
24
  secret_env = "CONFLUENCE_API_TOKEN"
24
25
 
25
26
  [capture]
@@ -60,8 +61,6 @@ entry = "/"
60
61
  api_path = "/__world-api"
61
62
  generated = false
62
63
 
63
- # ---------------------------------------------------------------- spaces
64
-
65
64
  [[operations]]
66
65
  name = "get_spaces"
67
66
  method = "GET"
@@ -96,8 +95,6 @@ filter = { id = "spaceId", status = "status", title = "title" }
96
95
  match = { title = "contains" }
97
96
  sample = { id = "65537" }
98
97
 
99
- # ---------------------------------------------------------------- pages
100
-
101
98
  [[operations]]
102
99
  name = "get_pages"
103
100
  method = "GET"
@@ -193,8 +190,6 @@ results = "results"
193
190
  filter = { id = "pageId" }
194
191
  sample = { id = "100063" }
195
192
 
196
- # ---------------------------------------------------------------- comments
197
-
198
193
  [[operations]]
199
194
  name = "get_page_footer_comments"
200
195
  method = "GET"
@@ -265,8 +260,6 @@ handler = "update_inline_comment"
265
260
  sample = { commentId = "cmt-005" }
266
261
  sample_body = { version = { number = 2 }, resolved = true }
267
262
 
268
- # ---------------------------------------------------------------- labels
269
-
270
263
  [[operations]]
271
264
  name = "get_labels"
272
265
  method = "GET"
@@ -289,8 +282,6 @@ action = "handler"
289
282
  handler = "get_page_labels"
290
283
  sample = { id = "100063" }
291
284
 
292
- # ---------------------------------------------------------------- the ui's own reads
293
- #
294
285
  # One route per call in the UI's api client. Every one is a handler: these are
295
286
  # composed read models (a page with its ancestors, children, comments and versions;
296
287
  # a space tree), not vendor routes, and no vendor document describes them.
@@ -417,8 +408,6 @@ results = ""
417
408
  action = "handler"
418
409
  handler = "ui_labels"
419
410
 
420
- # ---------------------------------------------------------------- the ui's writes
421
-
422
411
  [[operations]]
423
412
  name = "ui_add_comment"
424
413
  method = "POST"
@@ -16,6 +16,7 @@ base_url = "https://api.github.com"
16
16
 
17
17
  [auth]
18
18
  scheme = "bearer"
19
+ prefixes = ["Bearer", "token"]
19
20
  secret_env = "GITHUB_TOKEN"
20
21
 
21
22
  # email is the PII a capture sees on commit authors, users and organizations; `secret`
@@ -44,6 +44,7 @@ base_url = "https://instance.service-now.com"
44
44
  # is the other supported scheme and is registered per instance. No credential
45
45
  # ships with the template; the served world accepts the host's session credential.
46
46
  scheme = "basic"
47
+ credential = "password"
47
48
  secret_env = "SERVICENOW_BASIC_AUTH"
48
49
 
49
50
  [capture]
@@ -98,7 +99,7 @@ not_found_status = 404
98
99
  invalid_status = 400
99
100
 
100
101
  [ui]
101
- # Rishi's pixel-faithful Next Experience twin, built from twins/servicenow/ui with
102
+ # The Next Experience twin, built from twins/servicenow/ui with
102
103
  # no source edits. `generated = false`: ui/static is a build artifact of that
103
104
  # directory, not the generated page, so compile leaves it alone. The SPA calls the api at the same origin
104
105
  # under /__world-api, which the World Host rewrites onto the routes below.
@@ -109,8 +110,6 @@ entry = "/"
109
110
  api_path = "/__world-api"
110
111
  generated = false
111
112
 
112
- # ------------------------------------------------------- the Table API (vendor)
113
-
114
113
  [[operations]]
115
114
  name = "list_table_records"
116
115
  path = "/api/now/table/{tableName}"
@@ -192,8 +191,6 @@ handler = "delete_table_record"
192
191
  status = 204
193
192
  sample = { tableName = "incident_task", sys_id = "eaaefd673662a9417c46de3a53e6b147" }
194
193
 
195
- # ------------------------------- the Next Experience shell (what the ui calls)
196
-
197
194
  [[operations]]
198
195
  name = "get_me"
199
196
  path = "/api/now/ui/me"
@@ -0,0 +1,54 @@
1
+ """How a world answers browsers: ``[api] cors = true`` says the vendor's API serves pages on any
2
+ origin, so a preflight is answered and every answer names any origin; without it a world sends
3
+ no CORS header and an OPTIONS request is routed like any other method."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import re
8
+ from dataclasses import dataclass
9
+
10
+ ALLOW_ORIGIN = "Access-Control-Allow-Origin"
11
+ REQUESTED_HEADERS = "access-control-request-headers"
12
+ HEADER_NAMES = re.compile(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+(?:[ \t]*,[ \t]*[!#$%&'*+.^_`|~0-9A-Za-z-]+)*")
13
+ MAX_REQUESTED_CHARS = 1024
14
+
15
+ Preflight = tuple[int, None, dict[str, str]]
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Cors:
20
+ """A vendor whose API serves no browser."""
21
+
22
+ @staticmethod
23
+ def of(declared: bool, methods: tuple[str, ...]) -> Cors:
24
+ """The world's CORS: :class:`OpenCors` over ``methods`` when declared, else none."""
25
+ return OpenCors(methods) if declared else Cors()
26
+
27
+ def preflight(self, method: str, headers: dict[str, str]) -> Preflight | None:
28
+ return None
29
+
30
+ def carried(self, extra: dict[str, str] | None) -> dict[str, str] | None:
31
+ return extra
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class OpenCors(Cors):
36
+ """A vendor whose API answers pages on any origin, without credentials. A preflight is
37
+ answered before any credential is checked, since a browser sends none with it: it names the
38
+ methods the world serves and, when they are a list of header names, the headers the page
39
+ asked to send. It reads nothing and writes nothing. Every other answer exposes its headers
40
+ (``Link``, the page headers) to the page."""
41
+
42
+ methods: tuple[str, ...]
43
+
44
+ def preflight(self, method: str, headers: dict[str, str]) -> Preflight | None:
45
+ if method != "OPTIONS":
46
+ return None
47
+ allowed = {ALLOW_ORIGIN: "*", "Access-Control-Allow-Methods": ", ".join(self.methods)}
48
+ requested = headers.get(REQUESTED_HEADERS, "")
49
+ if len(requested) <= MAX_REQUESTED_CHARS and HEADER_NAMES.fullmatch(requested):
50
+ allowed["Access-Control-Allow-Headers"] = requested
51
+ return 204, None, allowed
52
+
53
+ def carried(self, extra: dict[str, str] | None) -> dict[str, str] | None:
54
+ return {**(extra or {}), ALLOW_ORIGIN: "*", "Access-Control-Expose-Headers": "*"}