@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -10,10 +10,11 @@ connection is described once::
|
|
|
10
10
|
base_url = "https://api.spycloud.io/investigations-v2"
|
|
11
11
|
|
|
12
12
|
[auth]
|
|
13
|
-
scheme = "header" # header | bearer | basic | query | none
|
|
13
|
+
scheme = "header" # header | bearer | basic | query | oauth2 | none
|
|
14
14
|
name = "x-api-key" # header or query parameter that carries the secret
|
|
15
15
|
secret_env = "SPYCLOUD_API_KEY"
|
|
16
16
|
headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" } # more credential headers, each named by an env var
|
|
17
|
+
# [auth.login] and [auth.downloads]: the vendor's own login and signed links (door.py)
|
|
17
18
|
|
|
18
19
|
[capture]
|
|
19
20
|
requests_per_minute = 30 # pacing between requests; providers ask us to be sparing
|
|
@@ -58,7 +59,6 @@ the standard library, so the credential-free bridge can load it.
|
|
|
58
59
|
|
|
59
60
|
from __future__ import annotations
|
|
60
61
|
|
|
61
|
-
import base64
|
|
62
62
|
import hashlib
|
|
63
63
|
import json
|
|
64
64
|
import os
|
|
@@ -76,23 +76,34 @@ from typing import Any, Callable, Iterable
|
|
|
76
76
|
|
|
77
77
|
from .compiler import check_bundle, compile_bundle
|
|
78
78
|
from .database import WorldDatabase
|
|
79
|
+
from .door import VendorAuth
|
|
79
80
|
from .primary_key import PrimaryKey
|
|
80
81
|
from .transforms import apply_transforms
|
|
81
|
-
from .validation import
|
|
82
|
+
from .validation import (
|
|
83
|
+
FIELD_NAME,
|
|
84
|
+
METHODS,
|
|
85
|
+
PLACEHOLDER,
|
|
86
|
+
ContractError,
|
|
87
|
+
fail,
|
|
88
|
+
names_of,
|
|
89
|
+
operation_name,
|
|
90
|
+
parse_toml,
|
|
91
|
+
path_pattern,
|
|
92
|
+
route_path,
|
|
93
|
+
string_of,
|
|
94
|
+
table_of,
|
|
95
|
+
)
|
|
96
|
+
from .webhooks import Webhooks
|
|
82
97
|
|
|
83
98
|
SPEC = "connector.toml"
|
|
84
99
|
CAPTURES = "captures"
|
|
85
100
|
MANIFEST = "manifest.json"
|
|
86
|
-
SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
|
|
87
|
-
DEFAULT_TOKEN_PATH = "/oauth/token"
|
|
88
|
-
METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
89
101
|
SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
90
102
|
#: A declared query parameter as the vendor names it: dotted (``query.cond``,
|
|
91
103
|
#: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
|
|
92
104
|
#: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
|
|
93
105
|
#: match or filter on it.
|
|
94
106
|
PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
|
|
95
|
-
PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
96
107
|
# Pairs, not a dict, so a repeated key survives.
|
|
97
108
|
Params = list[tuple[str, str]]
|
|
98
109
|
MAX_BODY = 8 * 1024 * 1024
|
|
@@ -139,18 +150,21 @@ class Operation:
|
|
|
139
150
|
def placeholders(self) -> tuple[str, ...]:
|
|
140
151
|
return tuple(PLACEHOLDER.findall(self.path))
|
|
141
152
|
|
|
153
|
+
@property
|
|
154
|
+
def pattern(self) -> re.Pattern:
|
|
155
|
+
"""The route's path as a pattern over a request's path, each ``{arg}`` one segment."""
|
|
156
|
+
return path_pattern(self.path)
|
|
157
|
+
|
|
158
|
+
def path_args(self, path: str) -> dict[str, str]:
|
|
159
|
+
found = self.pattern.match(path)
|
|
160
|
+
return {k: urllib.parse.unquote(v) for k, v in found.groupdict().items()} if found else {}
|
|
161
|
+
|
|
142
162
|
|
|
143
163
|
@dataclass(frozen=True)
|
|
144
164
|
class Spec:
|
|
145
165
|
slug: str
|
|
146
166
|
base_url: str
|
|
147
|
-
|
|
148
|
-
auth_name: str | None
|
|
149
|
-
secret_env: str | None
|
|
150
|
-
extra_headers: dict[str, str]
|
|
151
|
-
token_url: str | None
|
|
152
|
-
client_id_env: str | None
|
|
153
|
-
scope: str | None
|
|
167
|
+
auth: VendorAuth
|
|
154
168
|
requests_per_minute: int
|
|
155
169
|
timeout_seconds: float
|
|
156
170
|
max_pages: int
|
|
@@ -162,6 +176,12 @@ class Spec:
|
|
|
162
176
|
round_fields: dict[str, int] = field(default_factory=dict)
|
|
163
177
|
preserve_fields: frozenset[str] = frozenset()
|
|
164
178
|
operations: dict[str, Operation] = field(default_factory=dict)
|
|
179
|
+
webhooks: Webhooks | None = None
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def scheme(self) -> str:
|
|
183
|
+
"""The ``[auth] scheme`` the connection declares."""
|
|
184
|
+
return self.auth.scheme.name
|
|
165
185
|
|
|
166
186
|
def operation(self, name: str) -> Operation:
|
|
167
187
|
if name not in self.operations:
|
|
@@ -190,23 +210,8 @@ class Spec:
|
|
|
190
210
|
"`gateway worlds schema entity add`, or point the route at one that exists",
|
|
191
211
|
location,
|
|
192
212
|
)
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
def _table(value: Any, location: str) -> dict:
|
|
196
|
-
if not isinstance(value, dict):
|
|
197
|
-
fail("spec", "Expected a table", location)
|
|
198
|
-
return value
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
|
|
202
|
-
if key not in table:
|
|
203
|
-
if required:
|
|
204
|
-
fail("spec", f"Missing {key}", location)
|
|
205
|
-
return default
|
|
206
|
-
value = table[key]
|
|
207
|
-
if not isinstance(value, str) or not value.strip():
|
|
208
|
-
fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
|
|
209
|
-
return value.strip()
|
|
213
|
+
if self.webhooks is not None:
|
|
214
|
+
self.webhooks.check_entity(known)
|
|
210
215
|
|
|
211
216
|
|
|
212
217
|
RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
|
|
@@ -216,7 +221,7 @@ def parse_redact(value: Any, location: str) -> RedactTable:
|
|
|
216
221
|
"""A ``[redact]`` table as ``(hash, drop, round, preserve)`` — the one grammar for
|
|
217
222
|
the policy wherever it is declared (``connector.toml`` for a vendor world,
|
|
218
223
|
``gateway-env.toml`` for a tools-only schema world)."""
|
|
219
|
-
redact =
|
|
224
|
+
redact = table_of(value, location)
|
|
220
225
|
hash_fields = frozenset(_paths(redact, "hash", location))
|
|
221
226
|
drop_fields = frozenset(_paths(redact, "drop", location))
|
|
222
227
|
rounding = redact.get("round", {})
|
|
@@ -260,15 +265,6 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
|
|
|
260
265
|
return verbs
|
|
261
266
|
|
|
262
267
|
|
|
263
|
-
def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
|
|
264
|
-
value = table.get(key, [])
|
|
265
|
-
if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
|
|
266
|
-
fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
|
|
267
|
-
if len(set(value)) != len(value):
|
|
268
|
-
fail("spec", f"{key} repeats a name", f"{location}/{key}")
|
|
269
|
-
return tuple(value)
|
|
270
|
-
|
|
271
|
-
|
|
272
268
|
def pairs_of(params: Any) -> Params:
|
|
273
269
|
"""Query parameters as an ordered list of ``(name, value)`` pairs.
|
|
274
270
|
|
|
@@ -315,81 +311,68 @@ def parse_spec(text: str) -> Spec:
|
|
|
315
311
|
|
|
316
312
|
def spec_from(raw: dict) -> Spec:
|
|
317
313
|
"""The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
|
|
318
|
-
connector =
|
|
319
|
-
slug =
|
|
314
|
+
connector = table_of(raw.get("connector"), "/connector")
|
|
315
|
+
slug = string_of(connector, "slug", "/connector")
|
|
320
316
|
if not SLUG.match(slug):
|
|
321
317
|
fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
|
|
322
|
-
base_url =
|
|
318
|
+
base_url = string_of(connector, "base_url", "/connector")
|
|
323
319
|
parsed = urllib.parse.urlsplit(base_url)
|
|
324
320
|
if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
|
|
325
321
|
fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
|
|
326
322
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
|
|
332
|
-
secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
|
|
333
|
-
if scheme == "none" and (auth_name or secret_env):
|
|
334
|
-
fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
|
|
335
|
-
token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
|
|
336
|
-
client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
|
|
337
|
-
scope = _string(auth, "scope", "/auth", required=False)
|
|
338
|
-
if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
|
|
339
|
-
fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
|
|
340
|
-
if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
|
|
341
|
-
fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
|
|
342
|
-
extra_headers = auth.get("headers", {})
|
|
343
|
-
if not isinstance(extra_headers, dict) or not all(
|
|
344
|
-
isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
|
|
345
|
-
):
|
|
346
|
-
fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
|
|
347
|
-
if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
|
|
348
|
-
fail("spec", "auth.headers repeats the secret header", "/auth/headers")
|
|
349
|
-
|
|
350
|
-
capture = _table(raw.get("capture", {}), "/capture")
|
|
351
|
-
cursor_param = _string(capture, "cursor_param", "/capture", required=False)
|
|
352
|
-
cursor_field = _string(capture, "cursor_field", "/capture", required=False)
|
|
353
|
-
cursor_header = _string(capture, "cursor_header", "/capture", required=False)
|
|
323
|
+
capture = table_of(raw.get("capture", {}), "/capture")
|
|
324
|
+
cursor_param = string_of(capture, "cursor_param", "/capture", required=False)
|
|
325
|
+
cursor_field = string_of(capture, "cursor_field", "/capture", required=False)
|
|
326
|
+
cursor_header = string_of(capture, "cursor_header", "/capture", required=False)
|
|
354
327
|
if cursor_field and cursor_header:
|
|
355
328
|
fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
|
|
356
329
|
if bool(cursor_param) != bool(cursor_field or cursor_header):
|
|
357
330
|
fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
|
|
358
331
|
|
|
359
332
|
hash_fields, drop_fields, round_fields, preserve_fields = parse_redact(raw.get("redact", {}), "/redact")
|
|
333
|
+
operations = _operations(raw.get("operations", []))
|
|
334
|
+
|
|
335
|
+
return Spec(
|
|
336
|
+
slug=slug,
|
|
337
|
+
base_url=base_url.rstrip("/"),
|
|
338
|
+
auth=VendorAuth.parse(raw.get("auth", {"scheme": "none"}), operations),
|
|
339
|
+
requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
|
|
340
|
+
timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
|
|
341
|
+
max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
|
|
342
|
+
cursor_param=cursor_param,
|
|
343
|
+
cursor_field=cursor_field,
|
|
344
|
+
cursor_header=cursor_header,
|
|
345
|
+
hash_fields=hash_fields,
|
|
346
|
+
drop_fields=drop_fields,
|
|
347
|
+
round_fields=round_fields,
|
|
348
|
+
preserve_fields=preserve_fields,
|
|
349
|
+
operations=operations,
|
|
350
|
+
webhooks=Webhooks.parse(raw.get("webhooks")),
|
|
351
|
+
)
|
|
360
352
|
|
|
353
|
+
|
|
354
|
+
def _operations(listed: Any) -> dict[str, Operation]:
|
|
355
|
+
"""Every ``[[operations]]`` entry by name; two that would answer the same request are refused."""
|
|
361
356
|
operations: dict[str, Operation] = {}
|
|
362
357
|
routes: dict[tuple[str, str, tuple[str, ...]], str] = {}
|
|
363
|
-
listed = raw.get("operations", [])
|
|
364
358
|
if not isinstance(listed, list) or not listed:
|
|
365
359
|
fail("spec", "Declare at least one [[operations]] entry", "/operations")
|
|
366
360
|
for index, entry in enumerate(listed):
|
|
367
361
|
location = f"/operations/{index}"
|
|
368
|
-
table =
|
|
369
|
-
name =
|
|
362
|
+
table = table_of(entry, location)
|
|
363
|
+
name = string_of(table, "name", location)
|
|
370
364
|
operation_name(name, f"{location}/name")
|
|
371
365
|
if name in operations:
|
|
372
366
|
fail("spec", f"operation {name} is declared twice", f"{location}/name")
|
|
373
367
|
methods = _verbs(table, location)
|
|
374
|
-
method =
|
|
368
|
+
method = string_of(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
|
|
375
369
|
if method not in METHODS:
|
|
376
370
|
fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
|
|
377
371
|
if methods and method not in methods:
|
|
378
372
|
fail("spec", f"method {method} must be one of methods ({', '.join(methods)})", f"{location}/method")
|
|
379
|
-
path =
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
placeholders = PLACEHOLDER.findall(path)
|
|
383
|
-
repeated = [p for p in placeholders if placeholders.count(p) > 1]
|
|
384
|
-
if repeated:
|
|
385
|
-
fail(
|
|
386
|
-
"spec",
|
|
387
|
-
f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
|
|
388
|
-
f"{location}/path",
|
|
389
|
-
argument=repeated[0],
|
|
390
|
-
)
|
|
391
|
-
params = _names(table, "params", location, PARAM)
|
|
392
|
-
entity = _string(table, "entity", location, required=False)
|
|
373
|
+
path = route_path(string_of(table, "path", location) or "", f"{location}/path")
|
|
374
|
+
params = names_of(table, "params", location, PARAM)
|
|
375
|
+
entity = string_of(table, "entity", location, required=False)
|
|
393
376
|
results = table.get("results", "")
|
|
394
377
|
if not isinstance(results, str):
|
|
395
378
|
fail("spec", "results must be a dotted path string", f"{location}/results")
|
|
@@ -416,29 +399,7 @@ def spec_from(raw: dict) -> Spec:
|
|
|
416
399
|
f"{location}/path",
|
|
417
400
|
)
|
|
418
401
|
routes[route] = name
|
|
419
|
-
|
|
420
|
-
return Spec(
|
|
421
|
-
slug=slug,
|
|
422
|
-
base_url=base_url.rstrip("/"),
|
|
423
|
-
scheme=scheme,
|
|
424
|
-
auth_name=auth_name,
|
|
425
|
-
secret_env=secret_env,
|
|
426
|
-
extra_headers=dict(extra_headers),
|
|
427
|
-
token_url=token_url if scheme == "oauth2" else None,
|
|
428
|
-
client_id_env=client_id_env if scheme == "oauth2" else None,
|
|
429
|
-
scope=scope if scheme == "oauth2" else None,
|
|
430
|
-
requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
|
|
431
|
-
timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
|
|
432
|
-
max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
|
|
433
|
-
cursor_param=cursor_param,
|
|
434
|
-
cursor_field=cursor_field,
|
|
435
|
-
cursor_header=cursor_header,
|
|
436
|
-
hash_fields=hash_fields,
|
|
437
|
-
drop_fields=drop_fields,
|
|
438
|
-
round_fields=round_fields,
|
|
439
|
-
preserve_fields=preserve_fields,
|
|
440
|
-
operations=operations,
|
|
441
|
-
)
|
|
402
|
+
return operations
|
|
442
403
|
|
|
443
404
|
|
|
444
405
|
def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
|
|
@@ -451,8 +412,8 @@ def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
|
|
|
451
412
|
|
|
452
413
|
|
|
453
414
|
def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...], dict[str, str]]:
|
|
454
|
-
table =
|
|
455
|
-
drop =
|
|
415
|
+
table = table_of(raw, location)
|
|
416
|
+
drop = names_of(table, "drop", location)
|
|
456
417
|
carry_args = _mapping(table, "carry_args", location)
|
|
457
418
|
explode: list[Explode] = []
|
|
458
419
|
listed = table.get("explode", [])
|
|
@@ -460,9 +421,9 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
|
|
|
460
421
|
fail("spec", "explode must be a list of tables", f"{location}/explode")
|
|
461
422
|
for index, entry in enumerate(listed):
|
|
462
423
|
where = f"{location}/explode/{index}"
|
|
463
|
-
item =
|
|
464
|
-
field_name =
|
|
465
|
-
entity =
|
|
424
|
+
item = table_of(entry, where)
|
|
425
|
+
field_name = string_of(item, "field", where)
|
|
426
|
+
entity = string_of(item, "entity", where)
|
|
466
427
|
carry = _mapping(item, "carry", where)
|
|
467
428
|
take = _mapping(item, "take", where)
|
|
468
429
|
if not carry and not take:
|
|
@@ -501,20 +462,18 @@ def load_spec(world: str | Path) -> Spec:
|
|
|
501
462
|
def _secret_value(spec: Spec, secret: str | None) -> str:
|
|
502
463
|
if secret is not None:
|
|
503
464
|
return secret
|
|
504
|
-
value = os.environ.get(spec.secret_env or "")
|
|
465
|
+
value = os.environ.get(spec.auth.secret_env or "")
|
|
505
466
|
if not value:
|
|
506
|
-
fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
|
|
467
|
+
fail("secret", f"Set {spec.auth.secret_env} to the connection's credential", "/auth/secret_env")
|
|
507
468
|
return value
|
|
508
469
|
|
|
509
470
|
|
|
510
471
|
def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
|
|
511
|
-
"""The value the requests carry: the credential itself, or
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
return mint_token(spec, secret=secret, opener=opener)
|
|
517
|
-
return _secret_value(spec, secret)
|
|
472
|
+
"""The value the requests carry: the credential itself, or the token the
|
|
473
|
+
scheme exchanges it for first (oauth2)."""
|
|
474
|
+
return spec.auth.scheme.capture_value(
|
|
475
|
+
lambda: _secret_value(spec, secret), lambda: mint_token(spec, secret=secret, opener=opener)
|
|
476
|
+
)
|
|
518
477
|
|
|
519
478
|
|
|
520
479
|
def build_request(
|
|
@@ -540,19 +499,14 @@ def build_request(
|
|
|
540
499
|
query = list(pairs_of(params))
|
|
541
500
|
recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
542
501
|
headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
|
|
543
|
-
for header, variable in spec.extra_headers.items():
|
|
502
|
+
for header, variable in spec.auth.extra_headers.items():
|
|
544
503
|
value = os.environ.get(variable)
|
|
545
504
|
if not value:
|
|
546
505
|
fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
|
|
547
506
|
headers[header] = value
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
headers["Authorization"] = f"Bearer {secret}"
|
|
552
|
-
elif spec.scheme == "basic":
|
|
553
|
-
headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
|
|
554
|
-
elif spec.scheme == "query":
|
|
555
|
-
query.append((spec.auth_name or "", secret or ""))
|
|
507
|
+
carried_headers, carried_query = spec.auth.scheme.carry(secret or "")
|
|
508
|
+
headers.update(carried_headers)
|
|
509
|
+
query.extend(carried_query.items())
|
|
556
510
|
url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
557
511
|
return urllib.request.Request(url, method=operation.method, headers=headers), recorded
|
|
558
512
|
|
|
@@ -567,29 +521,15 @@ def undeclared_params(spec: Spec, operation: Operation, params: Params) -> list[
|
|
|
567
521
|
return seen
|
|
568
522
|
|
|
569
523
|
|
|
570
|
-
def token_endpoint(spec: Spec) -> str:
|
|
571
|
-
"""Where an oauth2 connection exchanges its client credentials."""
|
|
572
|
-
url = spec.token_url or DEFAULT_TOKEN_PATH
|
|
573
|
-
return url if url.startswith(("https://", "http://")) else spec.base_url + url
|
|
574
|
-
|
|
575
|
-
|
|
576
524
|
def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
|
|
577
525
|
"""Exchange the client credentials for an access token (RFC 6749 §4.4).
|
|
578
526
|
|
|
579
527
|
The client id and secret come from the environment the way every other
|
|
580
528
|
credential does; the token is what the captures then send as a bearer.
|
|
581
529
|
Nothing about the exchange is recorded, so the secret never reaches disk."""
|
|
582
|
-
|
|
583
|
-
fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
|
|
584
|
-
client_id = os.environ.get(spec.client_id_env or "")
|
|
585
|
-
if not client_id:
|
|
586
|
-
fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
|
|
587
|
-
client_secret = _secret_value(spec, secret)
|
|
588
|
-
form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
|
|
589
|
-
if spec.scope:
|
|
590
|
-
form["scope"] = spec.scope
|
|
530
|
+
endpoint, form = spec.auth.scheme.token_request(spec.base_url, lambda: _secret_value(spec, secret), os.environ)
|
|
591
531
|
request = urllib.request.Request(
|
|
592
|
-
|
|
532
|
+
endpoint,
|
|
593
533
|
method="POST",
|
|
594
534
|
data=urllib.parse.urlencode(form).encode(),
|
|
595
535
|
headers={
|
|
@@ -808,7 +748,9 @@ def capture(
|
|
|
808
748
|
following = urllib.parse.urlsplit(cursor)
|
|
809
749
|
if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
|
|
810
750
|
fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
|
|
811
|
-
page_params = [
|
|
751
|
+
page_params = [
|
|
752
|
+
(k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != spec.auth.scheme.secret_param
|
|
753
|
+
]
|
|
812
754
|
elif cursor:
|
|
813
755
|
cursor_param = spec.cursor_param or ""
|
|
814
756
|
page_params = [(k, v) for k, v in page_params if k != cursor_param] + [(cursor_param, cursor)]
|
|
@@ -20,6 +20,7 @@ base_url = "https://your-domain.atlassian.net"
|
|
|
20
20
|
# UI's; a world served with no key accepts any non-empty one.
|
|
21
21
|
[auth]
|
|
22
22
|
scheme = "basic"
|
|
23
|
+
credential = "password"
|
|
23
24
|
secret_env = "CONFLUENCE_API_TOKEN"
|
|
24
25
|
|
|
25
26
|
[capture]
|
|
@@ -60,8 +61,6 @@ entry = "/"
|
|
|
60
61
|
api_path = "/__world-api"
|
|
61
62
|
generated = false
|
|
62
63
|
|
|
63
|
-
# ---------------------------------------------------------------- spaces
|
|
64
|
-
|
|
65
64
|
[[operations]]
|
|
66
65
|
name = "get_spaces"
|
|
67
66
|
method = "GET"
|
|
@@ -96,8 +95,6 @@ filter = { id = "spaceId", status = "status", title = "title" }
|
|
|
96
95
|
match = { title = "contains" }
|
|
97
96
|
sample = { id = "65537" }
|
|
98
97
|
|
|
99
|
-
# ---------------------------------------------------------------- pages
|
|
100
|
-
|
|
101
98
|
[[operations]]
|
|
102
99
|
name = "get_pages"
|
|
103
100
|
method = "GET"
|
|
@@ -193,8 +190,6 @@ results = "results"
|
|
|
193
190
|
filter = { id = "pageId" }
|
|
194
191
|
sample = { id = "100063" }
|
|
195
192
|
|
|
196
|
-
# ---------------------------------------------------------------- comments
|
|
197
|
-
|
|
198
193
|
[[operations]]
|
|
199
194
|
name = "get_page_footer_comments"
|
|
200
195
|
method = "GET"
|
|
@@ -265,8 +260,6 @@ handler = "update_inline_comment"
|
|
|
265
260
|
sample = { commentId = "cmt-005" }
|
|
266
261
|
sample_body = { version = { number = 2 }, resolved = true }
|
|
267
262
|
|
|
268
|
-
# ---------------------------------------------------------------- labels
|
|
269
|
-
|
|
270
263
|
[[operations]]
|
|
271
264
|
name = "get_labels"
|
|
272
265
|
method = "GET"
|
|
@@ -289,8 +282,6 @@ action = "handler"
|
|
|
289
282
|
handler = "get_page_labels"
|
|
290
283
|
sample = { id = "100063" }
|
|
291
284
|
|
|
292
|
-
# ---------------------------------------------------------------- the ui's own reads
|
|
293
|
-
#
|
|
294
285
|
# One route per call in the UI's api client. Every one is a handler: these are
|
|
295
286
|
# composed read models (a page with its ancestors, children, comments and versions;
|
|
296
287
|
# a space tree), not vendor routes, and no vendor document describes them.
|
|
@@ -417,8 +408,6 @@ results = ""
|
|
|
417
408
|
action = "handler"
|
|
418
409
|
handler = "ui_labels"
|
|
419
410
|
|
|
420
|
-
# ---------------------------------------------------------------- the ui's writes
|
|
421
|
-
|
|
422
411
|
[[operations]]
|
|
423
412
|
name = "ui_add_comment"
|
|
424
413
|
method = "POST"
|
|
@@ -33,7 +33,8 @@ hosted call into such a handler answers 400 {error: "handler", detail, handler}:
|
|
|
33
33
|
Rows per entity, as {entity: count}.
|
|
34
34
|
|
|
35
35
|
A handler that raises AttributeError, TypeError or NameError is answered 400 with
|
|
36
|
-
that structured body;
|
|
36
|
+
that structured body; `raise Refusal(404, body)` (from gateway_env) answers that 4xx
|
|
37
|
+
and body as the vendor's refusal; any other exception is answered 500 {error: "engine", ...}.
|
|
37
38
|
`schema init --connector custom --with-example-tool` writes the example in full.
|
|
38
39
|
"""
|
|
39
40
|
|