@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -10,10 +10,11 @@ connection is described once::
10
10
  base_url = "https://api.spycloud.io/investigations-v2"
11
11
 
12
12
  [auth]
13
- scheme = "header" # header | bearer | basic | query | none
13
+ scheme = "header" # header | bearer | basic | query | oauth2 | none
14
14
  name = "x-api-key" # header or query parameter that carries the secret
15
15
  secret_env = "SPYCLOUD_API_KEY"
16
16
  headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" } # more credential headers, each named by an env var
17
+ # [auth.login] and [auth.downloads]: the vendor's own login and signed links (door.py)
17
18
 
18
19
  [capture]
19
20
  requests_per_minute = 30 # pacing between requests; providers ask us to be sparing
@@ -58,7 +59,6 @@ the standard library, so the credential-free bridge can load it.
58
59
 
59
60
  from __future__ import annotations
60
61
 
61
- import base64
62
62
  import hashlib
63
63
  import json
64
64
  import os
@@ -75,27 +75,35 @@ from pathlib import Path
75
75
  from typing import Any, Callable, Iterable
76
76
 
77
77
  from .compiler import check_bundle, compile_bundle
78
- from .store import SQLiteStore
78
+ from .database import WorldDatabase
79
+ from .door import VendorAuth
80
+ from .primary_key import PrimaryKey
79
81
  from .transforms import apply_transforms
80
- from .validation import ContractError, fail, operation_name, parse_toml
82
+ from .validation import (
83
+ FIELD_NAME,
84
+ METHODS,
85
+ PLACEHOLDER,
86
+ ContractError,
87
+ fail,
88
+ names_of,
89
+ operation_name,
90
+ parse_toml,
91
+ path_pattern,
92
+ route_path,
93
+ string_of,
94
+ table_of,
95
+ )
81
96
 
82
97
  SPEC = "connector.toml"
83
98
  CAPTURES = "captures"
84
99
  MANIFEST = "manifest.json"
85
- SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
86
- DEFAULT_TOKEN_PATH = "/oauth/token"
87
- METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
88
100
  SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
89
- #: Vendor-named things — query parameters, path arguments, redacted fields — keep the
90
- #: vendor's casing (``startDate``, ``Limit``); only our own operation names are snake_case.
91
- FIELD = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
92
- #: A declared query parameter may end in ``[]``: vendors (Brex, Stripe, PHP-style APIs)
93
- #: document repeated keys as ``expand[]=a&expand[]=b``, and a world that cannot declare
94
- #: the name cannot capture, match or filter on it.
95
- PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*(\[\])?$")
96
- PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
97
- #: Query parameters as sent: an ordered list of (name, value) pairs, so a repeated
98
- #: key (``expand[]=a&expand[]=b``) survives; a dict is accepted wherever pairs are.
101
+ #: A declared query parameter as the vendor names it: dotted (``query.cond``,
102
+ #: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
103
+ #: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
104
+ #: match or filter on it.
105
+ PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
106
+ # Pairs, not a dict, so a repeated key survives.
99
107
  Params = list[tuple[str, str]]
100
108
  MAX_BODY = 8 * 1024 * 1024
101
109
  RECORDED_HEADERS = ("content-type", "retry-after", "x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-reset")
@@ -128,13 +136,9 @@ class Operation:
128
136
  results: str
129
137
  drop: tuple[str, ...] = ()
130
138
  explode: tuple[Explode, ...] = ()
131
- #: row field <- name of a path argument or query parameter of the capture. A cash
132
- #: statement only knows its account from the ``{id}`` it was fetched under (Brex
133
- #: retrospective); the captured row never carries it, so ingest stamps it on.
139
+ # A captured row may lack the path argument it was fetched under, so ingest stamps it on.
134
140
  carry_args: dict[str, str] = field(default_factory=dict)
135
- #: Every verb the served route answers (``methods = ["GET", "POST"]``); ``method``
136
- #: is one of them and the one a capture sends. Slack's Web API takes a read as
137
- #: GET or POST alike, and its clients post everything.
141
+ # Slack's Web API takes a read as GET or POST alike, and its clients post everything.
138
142
  methods: tuple[str, ...] = ()
139
143
 
140
144
  def __post_init__(self) -> None:
@@ -145,20 +149,21 @@ class Operation:
145
149
  def placeholders(self) -> tuple[str, ...]:
146
150
  return tuple(PLACEHOLDER.findall(self.path))
147
151
 
152
+ @property
153
+ def pattern(self) -> re.Pattern:
154
+ """The route's path as a pattern over a request's path, each ``{arg}`` one segment."""
155
+ return path_pattern(self.path)
156
+
157
+ def path_args(self, path: str) -> dict[str, str]:
158
+ found = self.pattern.match(path)
159
+ return {k: urllib.parse.unquote(v) for k, v in found.groupdict().items()} if found else {}
160
+
148
161
 
149
162
  @dataclass(frozen=True)
150
163
  class Spec:
151
164
  slug: str
152
165
  base_url: str
153
- scheme: str
154
- auth_name: str | None
155
- secret_env: str | None
156
- extra_headers: dict[str, str] # header name -> env var holding its value
157
- #: oauth2 only: where the client-credentials grant is exchanged (absolute, or a
158
- #: path under base_url), the env var naming the client id, and the scope asked for.
159
- token_url: str | None
160
- client_id_env: str | None
161
- scope: str | None
166
+ auth: VendorAuth
162
167
  requests_per_minute: int
163
168
  timeout_seconds: float
164
169
  max_pages: int
@@ -167,10 +172,15 @@ class Spec:
167
172
  cursor_header: str | None
168
173
  hash_fields: frozenset[str]
169
174
  drop_fields: frozenset[str]
170
- round_fields: dict[str, int] = field(default_factory=dict) # numeric field -> decimals kept
171
- preserve_fields: frozenset[str] = frozenset() # hashed keeping length and character class
175
+ round_fields: dict[str, int] = field(default_factory=dict)
176
+ preserve_fields: frozenset[str] = frozenset()
172
177
  operations: dict[str, Operation] = field(default_factory=dict)
173
178
 
179
+ @property
180
+ def scheme(self) -> str:
181
+ """The ``[auth] scheme`` the connection declares."""
182
+ return self.auth.scheme.name
183
+
174
184
  def operation(self, name: str) -> Operation:
175
185
  if name not in self.operations:
176
186
  fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
@@ -200,26 +210,6 @@ class Spec:
200
210
  )
201
211
 
202
212
 
203
- # ---------------------------------------------------------------- spec
204
-
205
-
206
- def _table(value: Any, location: str) -> dict:
207
- if not isinstance(value, dict):
208
- fail("spec", "Expected a table", location)
209
- return value
210
-
211
-
212
- def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
213
- if key not in table:
214
- if required:
215
- fail("spec", f"Missing {key}", location)
216
- return default
217
- value = table[key]
218
- if not isinstance(value, str) or not value.strip():
219
- fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
220
- return value.strip()
221
-
222
-
223
213
  RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
224
214
 
225
215
 
@@ -227,12 +217,12 @@ def parse_redact(value: Any, location: str) -> RedactTable:
227
217
  """A ``[redact]`` table as ``(hash, drop, round, preserve)`` — the one grammar for
228
218
  the policy wherever it is declared (``connector.toml`` for a vendor world,
229
219
  ``gateway-env.toml`` for a tools-only schema world)."""
230
- redact = _table(value, location)
220
+ redact = table_of(value, location)
231
221
  hash_fields = frozenset(_paths(redact, "hash", location))
232
222
  drop_fields = frozenset(_paths(redact, "drop", location))
233
223
  rounding = redact.get("round", {})
234
224
  if not isinstance(rounding, dict) or not all(
235
- isinstance(k, str) and FIELD.match(k) and isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= 10
225
+ isinstance(k, str) and FIELD_NAME.match(k) and isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= 10
236
226
  for k, v in rounding.items()
237
227
  ):
238
228
  fail("spec", "redact.round maps numeric field names to the decimals kept (0-10)", f"{location}/round")
@@ -249,7 +239,7 @@ def _paths(table: dict, key: str, location: str) -> tuple[str, ...]:
249
239
  catalog's ``assets`` count map keeps its ``email`` key while records lose theirs."""
250
240
  value = table.get(key, [])
251
241
  if not isinstance(value, list) or not all(
252
- isinstance(v, str) and all(part == "*" or FIELD.match(part) for part in v.split(".")) and v for v in value
242
+ isinstance(v, str) and all(part == "*" or FIELD_NAME.match(part) for part in v.split(".")) and v for v in value
253
243
  ):
254
244
  fail("spec", f"{key} must be a list of field names or dotted paths", f"{location}/{key}")
255
245
  return tuple(value)
@@ -271,15 +261,6 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
271
261
  return verbs
272
262
 
273
263
 
274
- def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD) -> tuple[str, ...]:
275
- value = table.get(key, [])
276
- if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
277
- fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
278
- if len(set(value)) != len(value):
279
- fail("spec", f"{key} repeats a name", f"{location}/{key}")
280
- return tuple(value)
281
-
282
-
283
264
  def pairs_of(params: Any) -> Params:
284
265
  """Query parameters as an ordered list of ``(name, value)`` pairs.
285
266
 
@@ -326,81 +307,67 @@ def parse_spec(text: str) -> Spec:
326
307
 
327
308
  def spec_from(raw: dict) -> Spec:
328
309
  """The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
329
- connector = _table(raw.get("connector"), "/connector")
330
- slug = _string(connector, "slug", "/connector")
310
+ connector = table_of(raw.get("connector"), "/connector")
311
+ slug = string_of(connector, "slug", "/connector")
331
312
  if not SLUG.match(slug):
332
313
  fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
333
- base_url = _string(connector, "base_url", "/connector")
314
+ base_url = string_of(connector, "base_url", "/connector")
334
315
  parsed = urllib.parse.urlsplit(base_url)
335
316
  if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
336
317
  fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
337
318
 
338
- auth = _table(raw.get("auth", {"scheme": "none"}), "/auth")
339
- scheme = _string(auth, "scheme", "/auth")
340
- if scheme not in SCHEMES:
341
- fail("spec", f"auth.scheme must be one of {', '.join(SCHEMES)}", "/auth/scheme")
342
- auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
343
- secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
344
- if scheme == "none" and (auth_name or secret_env):
345
- fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
346
- token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
347
- client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
348
- scope = _string(auth, "scope", "/auth", required=False)
349
- if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
350
- fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
351
- if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
352
- fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
353
- extra_headers = auth.get("headers", {})
354
- if not isinstance(extra_headers, dict) or not all(
355
- isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
356
- ):
357
- fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
358
- if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
359
- fail("spec", "auth.headers repeats the secret header", "/auth/headers")
360
-
361
- capture = _table(raw.get("capture", {}), "/capture")
362
- cursor_param = _string(capture, "cursor_param", "/capture", required=False)
363
- cursor_field = _string(capture, "cursor_field", "/capture", required=False)
364
- cursor_header = _string(capture, "cursor_header", "/capture", required=False)
319
+ capture = table_of(raw.get("capture", {}), "/capture")
320
+ cursor_param = string_of(capture, "cursor_param", "/capture", required=False)
321
+ cursor_field = string_of(capture, "cursor_field", "/capture", required=False)
322
+ cursor_header = string_of(capture, "cursor_header", "/capture", required=False)
365
323
  if cursor_field and cursor_header:
366
324
  fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
367
325
  if bool(cursor_param) != bool(cursor_field or cursor_header):
368
326
  fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
369
327
 
370
328
  hash_fields, drop_fields, round_fields, preserve_fields = parse_redact(raw.get("redact", {}), "/redact")
329
+ operations = _operations(raw.get("operations", []))
330
+
331
+ return Spec(
332
+ slug=slug,
333
+ base_url=base_url.rstrip("/"),
334
+ auth=VendorAuth.parse(raw.get("auth", {"scheme": "none"}), operations),
335
+ requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
336
+ timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
337
+ max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
338
+ cursor_param=cursor_param,
339
+ cursor_field=cursor_field,
340
+ cursor_header=cursor_header,
341
+ hash_fields=hash_fields,
342
+ drop_fields=drop_fields,
343
+ round_fields=round_fields,
344
+ preserve_fields=preserve_fields,
345
+ operations=operations,
346
+ )
347
+
371
348
 
349
+ def _operations(listed: Any) -> dict[str, Operation]:
350
+ """Every ``[[operations]]`` entry by name; two that would answer the same request are refused."""
372
351
  operations: dict[str, Operation] = {}
373
352
  routes: dict[tuple[str, str, tuple[str, ...]], str] = {}
374
- listed = raw.get("operations", [])
375
353
  if not isinstance(listed, list) or not listed:
376
354
  fail("spec", "Declare at least one [[operations]] entry", "/operations")
377
355
  for index, entry in enumerate(listed):
378
356
  location = f"/operations/{index}"
379
- table = _table(entry, location)
380
- name = _string(table, "name", location)
357
+ table = table_of(entry, location)
358
+ name = string_of(table, "name", location)
381
359
  operation_name(name, f"{location}/name")
382
360
  if name in operations:
383
361
  fail("spec", f"operation {name} is declared twice", f"{location}/name")
384
362
  methods = _verbs(table, location)
385
- method = _string(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
363
+ method = string_of(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
386
364
  if method not in METHODS:
387
365
  fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
388
366
  if methods and method not in methods:
389
367
  fail("spec", f"method {method} must be one of methods ({', '.join(methods)})", f"{location}/method")
390
- path = _string(table, "path", location)
391
- if not path.startswith("/") or "?" in path or "#" in path:
392
- fail("spec", "path must start with / and carry no query", f"{location}/path")
393
- placeholders = PLACEHOLDER.findall(path)
394
- repeated = [p for p in placeholders if placeholders.count(p) > 1]
395
- if repeated:
396
- fail(
397
- "spec",
398
- f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
399
- f"{location}/path",
400
- argument=repeated[0],
401
- )
402
- params = _names(table, "params", location, PARAM)
403
- entity = _string(table, "entity", location, required=False)
368
+ path = route_path(string_of(table, "path", location) or "", f"{location}/path")
369
+ params = names_of(table, "params", location, PARAM)
370
+ entity = string_of(table, "entity", location, required=False)
404
371
  results = table.get("results", "")
405
372
  if not isinstance(results, str):
406
373
  fail("spec", "results must be a dotted path string", f"{location}/results")
@@ -427,29 +394,7 @@ def spec_from(raw: dict) -> Spec:
427
394
  f"{location}/path",
428
395
  )
429
396
  routes[route] = name
430
-
431
- return Spec(
432
- slug=slug,
433
- base_url=base_url.rstrip("/"),
434
- scheme=scheme,
435
- auth_name=auth_name,
436
- secret_env=secret_env,
437
- extra_headers=dict(extra_headers),
438
- token_url=token_url if scheme == "oauth2" else None,
439
- client_id_env=client_id_env if scheme == "oauth2" else None,
440
- scope=scope if scheme == "oauth2" else None,
441
- requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
442
- timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
443
- max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
444
- cursor_param=cursor_param,
445
- cursor_field=cursor_field,
446
- cursor_header=cursor_header,
447
- hash_fields=hash_fields,
448
- drop_fields=drop_fields,
449
- round_fields=round_fields,
450
- preserve_fields=preserve_fields,
451
- operations=operations,
452
- )
397
+ return operations
453
398
 
454
399
 
455
400
  def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
@@ -462,8 +407,8 @@ def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
462
407
 
463
408
 
464
409
  def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...], dict[str, str]]:
465
- table = _table(raw, location)
466
- drop = _names(table, "drop", location)
410
+ table = table_of(raw, location)
411
+ drop = names_of(table, "drop", location)
467
412
  carry_args = _mapping(table, "carry_args", location)
468
413
  explode: list[Explode] = []
469
414
  listed = table.get("explode", [])
@@ -471,9 +416,9 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
471
416
  fail("spec", "explode must be a list of tables", f"{location}/explode")
472
417
  for index, entry in enumerate(listed):
473
418
  where = f"{location}/explode/{index}"
474
- item = _table(entry, where)
475
- field_name = _string(item, "field", where)
476
- entity = _string(item, "entity", where)
419
+ item = table_of(entry, where)
420
+ field_name = string_of(item, "field", where)
421
+ entity = string_of(item, "entity", where)
477
422
  carry = _mapping(item, "carry", where)
478
423
  take = _mapping(item, "take", where)
479
424
  if not carry and not take:
@@ -509,26 +454,21 @@ def load_spec(world: str | Path) -> Spec:
509
454
  return bind_spec(spec, read_json(world_json))
510
455
 
511
456
 
512
- # ---------------------------------------------------------------- requests
513
-
514
-
515
457
  def _secret_value(spec: Spec, secret: str | None) -> str:
516
458
  if secret is not None:
517
459
  return secret
518
- value = os.environ.get(spec.secret_env or "")
460
+ value = os.environ.get(spec.auth.secret_env or "")
519
461
  if not value:
520
- fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
462
+ fail("secret", f"Set {spec.auth.secret_env} to the connection's credential", "/auth/secret_env")
521
463
  return value
522
464
 
523
465
 
524
466
  def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
525
- """The value the requests carry: the credential itself, or for oauth2 the
526
- access token minted from it."""
527
- if spec.scheme == "none":
528
- return None
529
- if spec.scheme == "oauth2":
530
- return mint_token(spec, secret=secret, opener=opener)
531
- return _secret_value(spec, secret)
467
+ """The value the requests carry: the credential itself, or the token the
468
+ scheme exchanges it for first (oauth2)."""
469
+ return spec.auth.scheme.capture_value(
470
+ lambda: _secret_value(spec, secret), lambda: mint_token(spec, secret=secret, opener=opener)
471
+ )
532
472
 
533
473
 
534
474
  def build_request(
@@ -554,19 +494,14 @@ def build_request(
554
494
  query = list(pairs_of(params))
555
495
  recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
556
496
  headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
557
- for header, variable in spec.extra_headers.items():
497
+ for header, variable in spec.auth.extra_headers.items():
558
498
  value = os.environ.get(variable)
559
499
  if not value:
560
500
  fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
561
501
  headers[header] = value
562
- if spec.scheme == "header":
563
- headers[spec.auth_name or ""] = secret or ""
564
- elif spec.scheme in ("bearer", "oauth2"):
565
- headers["Authorization"] = f"Bearer {secret}"
566
- elif spec.scheme == "basic":
567
- headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
568
- elif spec.scheme == "query":
569
- query.append((spec.auth_name or "", secret or ""))
502
+ carried_headers, carried_query = spec.auth.scheme.carry(secret or "")
503
+ headers.update(carried_headers)
504
+ query.extend(carried_query.items())
570
505
  url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
571
506
  return urllib.request.Request(url, method=operation.method, headers=headers), recorded
572
507
 
@@ -581,29 +516,15 @@ def undeclared_params(spec: Spec, operation: Operation, params: Params) -> list[
581
516
  return seen
582
517
 
583
518
 
584
- def token_endpoint(spec: Spec) -> str:
585
- """Where an oauth2 connection exchanges its client credentials."""
586
- url = spec.token_url or DEFAULT_TOKEN_PATH
587
- return url if url.startswith(("https://", "http://")) else spec.base_url + url
588
-
589
-
590
519
  def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
591
520
  """Exchange the client credentials for an access token (RFC 6749 §4.4).
592
521
 
593
522
  The client id and secret come from the environment the way every other
594
523
  credential does; the token is what the captures then send as a bearer.
595
524
  Nothing about the exchange is recorded, so the secret never reaches disk."""
596
- if spec.scheme != "oauth2":
597
- fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
598
- client_id = os.environ.get(spec.client_id_env or "")
599
- if not client_id:
600
- fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
601
- client_secret = _secret_value(spec, secret)
602
- form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
603
- if spec.scope:
604
- form["scope"] = spec.scope
525
+ endpoint, form = spec.auth.scheme.token_request(spec.base_url, lambda: _secret_value(spec, secret), os.environ)
605
526
  request = urllib.request.Request(
606
- token_endpoint(spec),
527
+ endpoint,
607
528
  method="POST",
608
529
  data=urllib.parse.urlencode(form).encode(),
609
530
  headers={
@@ -633,9 +554,6 @@ def _open(request: urllib.request.Request, timeout: float) -> Response:
633
554
  fail("transport", f"Could not reach the connection: {error}", "/capture")
634
555
 
635
556
 
636
- # ---------------------------------------------------------------- redaction
637
-
638
-
639
557
  def _named(key: str, here: str, names: frozenset[str]) -> bool:
640
558
  return key in names or here in names
641
559
 
@@ -759,7 +677,6 @@ def redact(value: Any, spec: Spec, path: str = "") -> Any:
759
677
  elif _named(key, here, spec.preserve_fields) and item is not None:
760
678
  out[key] = _shaped(spec.slug, item)
761
679
  elif key in spec.round_fields and isinstance(item, (int, float)) and not isinstance(item, bool):
762
- # Coordinates and the like: keep the shape and the type, lose the precision.
763
680
  out[key] = round(item, spec.round_fields[key])
764
681
  else:
765
682
  out[key] = redact(item, spec, here)
@@ -777,9 +694,6 @@ def _dig(body: Any, path: str) -> Any:
777
694
  return body
778
695
 
779
696
 
780
- # ---------------------------------------------------------------- capture
781
-
782
-
783
697
  def capture(
784
698
  world: str | Path,
785
699
  op: str,
@@ -825,12 +739,13 @@ def capture(
825
739
  sleep(pause)
826
740
  page_params = list(params)
827
741
  if cursor and cursor.startswith(("https://", "http://")):
828
- # A vendor that hands back the next page as a URL (DRF's ``next``): follow its
829
- # query, never its host — the page must come from the connection described.
742
+ # Follow a next-page URL's query, never its host: the page must come from the connection described.
830
743
  following = urllib.parse.urlsplit(cursor)
831
744
  if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
832
745
  fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
833
- page_params = [(k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != (spec.auth_name or "")]
746
+ page_params = [
747
+ (k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != spec.auth.scheme.secret_param
748
+ ]
834
749
  elif cursor:
835
750
  cursor_param = spec.cursor_param or ""
836
751
  page_params = [(k, v) for k, v in page_params if k != cursor_param] + [(cursor_param, cursor)]
@@ -898,22 +813,12 @@ def _append_manifest(root: Path, path: Path, record: dict, sha256: str) -> None:
898
813
  manifest_path.write_text(json.dumps(entries, indent=1) + "\n")
899
814
 
900
815
 
901
- # ---------------------------------------------------------------- ingest
902
-
903
-
904
- def _key(contract: Any, entity: str, row: dict) -> tuple | None:
905
- fields = contract.entities.get(entity, {}).get("primary_key", [])
906
- if not fields or any(field not in row for field in fields):
907
- return None
908
- return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
909
-
910
-
911
816
  def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list[dict]:
912
817
  """Insert what is new, update what changed, skip what the world already holds."""
913
818
  held: dict[tuple[str, tuple], dict] = {}
914
819
  for mutation in prior:
915
820
  if mutation.get("op") in ("insert", "update") and isinstance(mutation.get("value"), dict):
916
- key = _key(contract, mutation["entity"], mutation["value"])
821
+ key = PrimaryKey.in_contract(contract, mutation["entity"]).identity(mutation["value"])
917
822
  if key is not None:
918
823
  held[(mutation["entity"], key)] = mutation["value"]
919
824
  mutations: list[dict] = []
@@ -926,7 +831,8 @@ def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list
926
831
  if token in seen:
927
832
  continue
928
833
  seen[token] = entry["file"]
929
- key = _key(contract, entity, row)
834
+ primary = PrimaryKey.in_contract(contract, entity)
835
+ key = primary.identity(row)
930
836
  existing = held.get((entity, key)) if key is not None else None
931
837
  if existing == row:
932
838
  continue
@@ -940,8 +846,7 @@ def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list
940
846
  },
941
847
  }
942
848
  if existing is not None:
943
- fields = contract.entities[entity]["primary_key"]
944
- mutation["key"] = {field: row[field] for field in fields}
849
+ mutation["key"] = primary.key(row)
945
850
  mutations.append(mutation)
946
851
  return mutations
947
852
 
@@ -996,9 +901,7 @@ def response_rows(
996
901
  for raw_row in rows:
997
902
  if not isinstance(raw_row, dict):
998
903
  fail("ingest", f"A row in {where} is not an object", pointer)
999
- # The request is the authority for what the author asked to carry: a row that
1000
- # names the field itself is overwritten. A parameter the capture was made
1001
- # without is not stamped; the contract then says whether the row lacks it.
904
+ # The request is the authority for carried fields: a row naming the field itself is overwritten.
1002
905
  carried = {name: context[source] for name, source in operation.carry_args.items() if source in context}
1003
906
  out.extend(_project(operation, {**raw_row, **carried}, where, pointer))
1004
907
  return out
@@ -1074,17 +977,16 @@ def _prove(root: Path, contract: Any, batch: dict) -> dict[str, int]:
1074
977
  """Apply the batch in a scratch copy of the world; the entity counts it leaves, or the
1075
978
  store's refusal. The world on disk is never touched."""
1076
979
  from .._materialize import materialize_template # only ingest needs the parent package
980
+ from .snapshot import build_names
1077
981
 
1078
982
  with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
1079
983
  staged = Path(scratch) / "world"
1080
- shutil.copytree(
1081
- root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", "state.sqlite", "state.json")
1082
- )
984
+ shutil.copytree(root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", *build_names()))
1083
985
  (staged / "data/initial.json").write_text(json.dumps(batch, sort_keys=True, indent=1) + "\n")
1084
986
  compile_bundle(staged)
1085
987
  database = Path(scratch) / "world.db"
1086
988
  materialize_template(staged, database)
1087
- with SQLiteStore.open(database, check_bundle(staged)) as store:
989
+ with WorldDatabase.open(database, check_bundle(staged)) as store:
1088
990
  store.audit()
1089
991
  return {entity: len(store.list(entity)) for entity in contract.entities}
1090
992
 
@@ -1168,5 +1070,5 @@ def _counts(root: Path, contract: Any) -> dict[str, int]:
1168
1070
  with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
1169
1071
  database = Path(scratch) / "world.db"
1170
1072
  materialize_template(root, database)
1171
- with SQLiteStore.open(database, contract) as store:
1073
+ with WorldDatabase.open(database, contract) as store:
1172
1074
  return {entity: len(store.list(entity)) for entity in contract.entities}
@@ -20,6 +20,7 @@ base_url = "https://your-domain.atlassian.net"
20
20
  # UI's; a world served with no key accepts any non-empty one.
21
21
  [auth]
22
22
  scheme = "basic"
23
+ credential = "password"
23
24
  secret_env = "CONFLUENCE_API_TOKEN"
24
25
 
25
26
  [capture]
@@ -60,8 +61,6 @@ entry = "/"
60
61
  api_path = "/__world-api"
61
62
  generated = false
62
63
 
63
- # ---------------------------------------------------------------- spaces
64
-
65
64
  [[operations]]
66
65
  name = "get_spaces"
67
66
  method = "GET"
@@ -96,8 +95,6 @@ filter = { id = "spaceId", status = "status", title = "title" }
96
95
  match = { title = "contains" }
97
96
  sample = { id = "65537" }
98
97
 
99
- # ---------------------------------------------------------------- pages
100
-
101
98
  [[operations]]
102
99
  name = "get_pages"
103
100
  method = "GET"
@@ -193,8 +190,6 @@ results = "results"
193
190
  filter = { id = "pageId" }
194
191
  sample = { id = "100063" }
195
192
 
196
- # ---------------------------------------------------------------- comments
197
-
198
193
  [[operations]]
199
194
  name = "get_page_footer_comments"
200
195
  method = "GET"
@@ -265,8 +260,6 @@ handler = "update_inline_comment"
265
260
  sample = { commentId = "cmt-005" }
266
261
  sample_body = { version = { number = 2 }, resolved = true }
267
262
 
268
- # ---------------------------------------------------------------- labels
269
-
270
263
  [[operations]]
271
264
  name = "get_labels"
272
265
  method = "GET"
@@ -289,8 +282,6 @@ action = "handler"
289
282
  handler = "get_page_labels"
290
283
  sample = { id = "100063" }
291
284
 
292
- # ---------------------------------------------------------------- the ui's own reads
293
- #
294
285
  # One route per call in the UI's api client. Every one is a handler: these are
295
286
  # composed read models (a page with its ancestors, children, comments and versions;
296
287
  # a space tree), not vendor routes, and no vendor document describes them.
@@ -417,8 +408,6 @@ results = ""
417
408
  action = "handler"
418
409
  handler = "ui_labels"
419
410
 
420
- # ---------------------------------------------------------------- the ui's writes
421
-
422
411
  [[operations]]
423
412
  name = "ui_add_comment"
424
413
  method = "POST"
@@ -16,6 +16,7 @@ base_url = "https://api.github.com"
16
16
 
17
17
  [auth]
18
18
  scheme = "bearer"
19
+ prefixes = ["Bearer", "token"]
19
20
  secret_env = "GITHUB_TOKEN"
20
21
 
21
22
  # email is the PII a capture sees on commit authors, users and organizations; `secret`