@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -10,10 +10,11 @@ connection is described once::
|
|
|
10
10
|
base_url = "https://api.spycloud.io/investigations-v2"
|
|
11
11
|
|
|
12
12
|
[auth]
|
|
13
|
-
scheme = "header" # header | bearer | basic | query | none
|
|
13
|
+
scheme = "header" # header | bearer | basic | query | oauth2 | none
|
|
14
14
|
name = "x-api-key" # header or query parameter that carries the secret
|
|
15
15
|
secret_env = "SPYCLOUD_API_KEY"
|
|
16
16
|
headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" } # more credential headers, each named by an env var
|
|
17
|
+
# [auth.login] and [auth.downloads]: the vendor's own login and signed links (door.py)
|
|
17
18
|
|
|
18
19
|
[capture]
|
|
19
20
|
requests_per_minute = 30 # pacing between requests; providers ask us to be sparing
|
|
@@ -58,7 +59,6 @@ the standard library, so the credential-free bridge can load it.
|
|
|
58
59
|
|
|
59
60
|
from __future__ import annotations
|
|
60
61
|
|
|
61
|
-
import base64
|
|
62
62
|
import hashlib
|
|
63
63
|
import json
|
|
64
64
|
import os
|
|
@@ -76,23 +76,33 @@ from typing import Any, Callable, Iterable
|
|
|
76
76
|
|
|
77
77
|
from .compiler import check_bundle, compile_bundle
|
|
78
78
|
from .database import WorldDatabase
|
|
79
|
+
from .door import VendorAuth
|
|
79
80
|
from .primary_key import PrimaryKey
|
|
80
81
|
from .transforms import apply_transforms
|
|
81
|
-
from .validation import
|
|
82
|
+
from .validation import (
|
|
83
|
+
FIELD_NAME,
|
|
84
|
+
METHODS,
|
|
85
|
+
PLACEHOLDER,
|
|
86
|
+
ContractError,
|
|
87
|
+
fail,
|
|
88
|
+
names_of,
|
|
89
|
+
operation_name,
|
|
90
|
+
parse_toml,
|
|
91
|
+
path_pattern,
|
|
92
|
+
route_path,
|
|
93
|
+
string_of,
|
|
94
|
+
table_of,
|
|
95
|
+
)
|
|
82
96
|
|
|
83
97
|
SPEC = "connector.toml"
|
|
84
98
|
CAPTURES = "captures"
|
|
85
99
|
MANIFEST = "manifest.json"
|
|
86
|
-
SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
|
|
87
|
-
DEFAULT_TOKEN_PATH = "/oauth/token"
|
|
88
|
-
METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
89
100
|
SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
90
101
|
#: A declared query parameter as the vendor names it: dotted (``query.cond``,
|
|
91
102
|
#: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
|
|
92
103
|
#: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
|
|
93
104
|
#: match or filter on it.
|
|
94
105
|
PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
|
|
95
|
-
PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
96
106
|
# Pairs, not a dict, so a repeated key survives.
|
|
97
107
|
Params = list[tuple[str, str]]
|
|
98
108
|
MAX_BODY = 8 * 1024 * 1024
|
|
@@ -139,18 +149,21 @@ class Operation:
|
|
|
139
149
|
def placeholders(self) -> tuple[str, ...]:
|
|
140
150
|
return tuple(PLACEHOLDER.findall(self.path))
|
|
141
151
|
|
|
152
|
+
@property
|
|
153
|
+
def pattern(self) -> re.Pattern:
|
|
154
|
+
"""The route's path as a pattern over a request's path, each ``{arg}`` one segment."""
|
|
155
|
+
return path_pattern(self.path)
|
|
156
|
+
|
|
157
|
+
def path_args(self, path: str) -> dict[str, str]:
|
|
158
|
+
found = self.pattern.match(path)
|
|
159
|
+
return {k: urllib.parse.unquote(v) for k, v in found.groupdict().items()} if found else {}
|
|
160
|
+
|
|
142
161
|
|
|
143
162
|
@dataclass(frozen=True)
|
|
144
163
|
class Spec:
|
|
145
164
|
slug: str
|
|
146
165
|
base_url: str
|
|
147
|
-
|
|
148
|
-
auth_name: str | None
|
|
149
|
-
secret_env: str | None
|
|
150
|
-
extra_headers: dict[str, str]
|
|
151
|
-
token_url: str | None
|
|
152
|
-
client_id_env: str | None
|
|
153
|
-
scope: str | None
|
|
166
|
+
auth: VendorAuth
|
|
154
167
|
requests_per_minute: int
|
|
155
168
|
timeout_seconds: float
|
|
156
169
|
max_pages: int
|
|
@@ -163,6 +176,11 @@ class Spec:
|
|
|
163
176
|
preserve_fields: frozenset[str] = frozenset()
|
|
164
177
|
operations: dict[str, Operation] = field(default_factory=dict)
|
|
165
178
|
|
|
179
|
+
@property
|
|
180
|
+
def scheme(self) -> str:
|
|
181
|
+
"""The ``[auth] scheme`` the connection declares."""
|
|
182
|
+
return self.auth.scheme.name
|
|
183
|
+
|
|
166
184
|
def operation(self, name: str) -> Operation:
|
|
167
185
|
if name not in self.operations:
|
|
168
186
|
fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
|
|
@@ -192,23 +210,6 @@ class Spec:
|
|
|
192
210
|
)
|
|
193
211
|
|
|
194
212
|
|
|
195
|
-
def _table(value: Any, location: str) -> dict:
|
|
196
|
-
if not isinstance(value, dict):
|
|
197
|
-
fail("spec", "Expected a table", location)
|
|
198
|
-
return value
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
|
|
202
|
-
if key not in table:
|
|
203
|
-
if required:
|
|
204
|
-
fail("spec", f"Missing {key}", location)
|
|
205
|
-
return default
|
|
206
|
-
value = table[key]
|
|
207
|
-
if not isinstance(value, str) or not value.strip():
|
|
208
|
-
fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
|
|
209
|
-
return value.strip()
|
|
210
|
-
|
|
211
|
-
|
|
212
213
|
RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
|
|
213
214
|
|
|
214
215
|
|
|
@@ -216,7 +217,7 @@ def parse_redact(value: Any, location: str) -> RedactTable:
|
|
|
216
217
|
"""A ``[redact]`` table as ``(hash, drop, round, preserve)`` — the one grammar for
|
|
217
218
|
the policy wherever it is declared (``connector.toml`` for a vendor world,
|
|
218
219
|
``gateway-env.toml`` for a tools-only schema world)."""
|
|
219
|
-
redact =
|
|
220
|
+
redact = table_of(value, location)
|
|
220
221
|
hash_fields = frozenset(_paths(redact, "hash", location))
|
|
221
222
|
drop_fields = frozenset(_paths(redact, "drop", location))
|
|
222
223
|
rounding = redact.get("round", {})
|
|
@@ -260,15 +261,6 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
|
|
|
260
261
|
return verbs
|
|
261
262
|
|
|
262
263
|
|
|
263
|
-
def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
|
|
264
|
-
value = table.get(key, [])
|
|
265
|
-
if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
|
|
266
|
-
fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
|
|
267
|
-
if len(set(value)) != len(value):
|
|
268
|
-
fail("spec", f"{key} repeats a name", f"{location}/{key}")
|
|
269
|
-
return tuple(value)
|
|
270
|
-
|
|
271
|
-
|
|
272
264
|
def pairs_of(params: Any) -> Params:
|
|
273
265
|
"""Query parameters as an ordered list of ``(name, value)`` pairs.
|
|
274
266
|
|
|
@@ -315,81 +307,67 @@ def parse_spec(text: str) -> Spec:
|
|
|
315
307
|
|
|
316
308
|
def spec_from(raw: dict) -> Spec:
|
|
317
309
|
"""The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
|
|
318
|
-
connector =
|
|
319
|
-
slug =
|
|
310
|
+
connector = table_of(raw.get("connector"), "/connector")
|
|
311
|
+
slug = string_of(connector, "slug", "/connector")
|
|
320
312
|
if not SLUG.match(slug):
|
|
321
313
|
fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
|
|
322
|
-
base_url =
|
|
314
|
+
base_url = string_of(connector, "base_url", "/connector")
|
|
323
315
|
parsed = urllib.parse.urlsplit(base_url)
|
|
324
316
|
if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
|
|
325
317
|
fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
|
|
326
318
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
|
|
332
|
-
secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
|
|
333
|
-
if scheme == "none" and (auth_name or secret_env):
|
|
334
|
-
fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
|
|
335
|
-
token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
|
|
336
|
-
client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
|
|
337
|
-
scope = _string(auth, "scope", "/auth", required=False)
|
|
338
|
-
if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
|
|
339
|
-
fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
|
|
340
|
-
if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
|
|
341
|
-
fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
|
|
342
|
-
extra_headers = auth.get("headers", {})
|
|
343
|
-
if not isinstance(extra_headers, dict) or not all(
|
|
344
|
-
isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
|
|
345
|
-
):
|
|
346
|
-
fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
|
|
347
|
-
if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
|
|
348
|
-
fail("spec", "auth.headers repeats the secret header", "/auth/headers")
|
|
349
|
-
|
|
350
|
-
capture = _table(raw.get("capture", {}), "/capture")
|
|
351
|
-
cursor_param = _string(capture, "cursor_param", "/capture", required=False)
|
|
352
|
-
cursor_field = _string(capture, "cursor_field", "/capture", required=False)
|
|
353
|
-
cursor_header = _string(capture, "cursor_header", "/capture", required=False)
|
|
319
|
+
capture = table_of(raw.get("capture", {}), "/capture")
|
|
320
|
+
cursor_param = string_of(capture, "cursor_param", "/capture", required=False)
|
|
321
|
+
cursor_field = string_of(capture, "cursor_field", "/capture", required=False)
|
|
322
|
+
cursor_header = string_of(capture, "cursor_header", "/capture", required=False)
|
|
354
323
|
if cursor_field and cursor_header:
|
|
355
324
|
fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
|
|
356
325
|
if bool(cursor_param) != bool(cursor_field or cursor_header):
|
|
357
326
|
fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
|
|
358
327
|
|
|
359
328
|
hash_fields, drop_fields, round_fields, preserve_fields = parse_redact(raw.get("redact", {}), "/redact")
|
|
329
|
+
operations = _operations(raw.get("operations", []))
|
|
330
|
+
|
|
331
|
+
return Spec(
|
|
332
|
+
slug=slug,
|
|
333
|
+
base_url=base_url.rstrip("/"),
|
|
334
|
+
auth=VendorAuth.parse(raw.get("auth", {"scheme": "none"}), operations),
|
|
335
|
+
requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
|
|
336
|
+
timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
|
|
337
|
+
max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
|
|
338
|
+
cursor_param=cursor_param,
|
|
339
|
+
cursor_field=cursor_field,
|
|
340
|
+
cursor_header=cursor_header,
|
|
341
|
+
hash_fields=hash_fields,
|
|
342
|
+
drop_fields=drop_fields,
|
|
343
|
+
round_fields=round_fields,
|
|
344
|
+
preserve_fields=preserve_fields,
|
|
345
|
+
operations=operations,
|
|
346
|
+
)
|
|
360
347
|
|
|
348
|
+
|
|
349
|
+
def _operations(listed: Any) -> dict[str, Operation]:
|
|
350
|
+
"""Every ``[[operations]]`` entry by name; two that would answer the same request are refused."""
|
|
361
351
|
operations: dict[str, Operation] = {}
|
|
362
352
|
routes: dict[tuple[str, str, tuple[str, ...]], str] = {}
|
|
363
|
-
listed = raw.get("operations", [])
|
|
364
353
|
if not isinstance(listed, list) or not listed:
|
|
365
354
|
fail("spec", "Declare at least one [[operations]] entry", "/operations")
|
|
366
355
|
for index, entry in enumerate(listed):
|
|
367
356
|
location = f"/operations/{index}"
|
|
368
|
-
table =
|
|
369
|
-
name =
|
|
357
|
+
table = table_of(entry, location)
|
|
358
|
+
name = string_of(table, "name", location)
|
|
370
359
|
operation_name(name, f"{location}/name")
|
|
371
360
|
if name in operations:
|
|
372
361
|
fail("spec", f"operation {name} is declared twice", f"{location}/name")
|
|
373
362
|
methods = _verbs(table, location)
|
|
374
|
-
method =
|
|
363
|
+
method = string_of(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
|
|
375
364
|
if method not in METHODS:
|
|
376
365
|
fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
|
|
377
366
|
if methods and method not in methods:
|
|
378
367
|
fail("spec", f"method {method} must be one of methods ({', '.join(methods)})", f"{location}/method")
|
|
379
|
-
path =
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
placeholders = PLACEHOLDER.findall(path)
|
|
383
|
-
repeated = [p for p in placeholders if placeholders.count(p) > 1]
|
|
384
|
-
if repeated:
|
|
385
|
-
fail(
|
|
386
|
-
"spec",
|
|
387
|
-
f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
|
|
388
|
-
f"{location}/path",
|
|
389
|
-
argument=repeated[0],
|
|
390
|
-
)
|
|
391
|
-
params = _names(table, "params", location, PARAM)
|
|
392
|
-
entity = _string(table, "entity", location, required=False)
|
|
368
|
+
path = route_path(string_of(table, "path", location) or "", f"{location}/path")
|
|
369
|
+
params = names_of(table, "params", location, PARAM)
|
|
370
|
+
entity = string_of(table, "entity", location, required=False)
|
|
393
371
|
results = table.get("results", "")
|
|
394
372
|
if not isinstance(results, str):
|
|
395
373
|
fail("spec", "results must be a dotted path string", f"{location}/results")
|
|
@@ -416,29 +394,7 @@ def spec_from(raw: dict) -> Spec:
|
|
|
416
394
|
f"{location}/path",
|
|
417
395
|
)
|
|
418
396
|
routes[route] = name
|
|
419
|
-
|
|
420
|
-
return Spec(
|
|
421
|
-
slug=slug,
|
|
422
|
-
base_url=base_url.rstrip("/"),
|
|
423
|
-
scheme=scheme,
|
|
424
|
-
auth_name=auth_name,
|
|
425
|
-
secret_env=secret_env,
|
|
426
|
-
extra_headers=dict(extra_headers),
|
|
427
|
-
token_url=token_url if scheme == "oauth2" else None,
|
|
428
|
-
client_id_env=client_id_env if scheme == "oauth2" else None,
|
|
429
|
-
scope=scope if scheme == "oauth2" else None,
|
|
430
|
-
requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
|
|
431
|
-
timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
|
|
432
|
-
max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
|
|
433
|
-
cursor_param=cursor_param,
|
|
434
|
-
cursor_field=cursor_field,
|
|
435
|
-
cursor_header=cursor_header,
|
|
436
|
-
hash_fields=hash_fields,
|
|
437
|
-
drop_fields=drop_fields,
|
|
438
|
-
round_fields=round_fields,
|
|
439
|
-
preserve_fields=preserve_fields,
|
|
440
|
-
operations=operations,
|
|
441
|
-
)
|
|
397
|
+
return operations
|
|
442
398
|
|
|
443
399
|
|
|
444
400
|
def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
|
|
@@ -451,8 +407,8 @@ def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
|
|
|
451
407
|
|
|
452
408
|
|
|
453
409
|
def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...], dict[str, str]]:
|
|
454
|
-
table =
|
|
455
|
-
drop =
|
|
410
|
+
table = table_of(raw, location)
|
|
411
|
+
drop = names_of(table, "drop", location)
|
|
456
412
|
carry_args = _mapping(table, "carry_args", location)
|
|
457
413
|
explode: list[Explode] = []
|
|
458
414
|
listed = table.get("explode", [])
|
|
@@ -460,9 +416,9 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
|
|
|
460
416
|
fail("spec", "explode must be a list of tables", f"{location}/explode")
|
|
461
417
|
for index, entry in enumerate(listed):
|
|
462
418
|
where = f"{location}/explode/{index}"
|
|
463
|
-
item =
|
|
464
|
-
field_name =
|
|
465
|
-
entity =
|
|
419
|
+
item = table_of(entry, where)
|
|
420
|
+
field_name = string_of(item, "field", where)
|
|
421
|
+
entity = string_of(item, "entity", where)
|
|
466
422
|
carry = _mapping(item, "carry", where)
|
|
467
423
|
take = _mapping(item, "take", where)
|
|
468
424
|
if not carry and not take:
|
|
@@ -501,20 +457,18 @@ def load_spec(world: str | Path) -> Spec:
|
|
|
501
457
|
def _secret_value(spec: Spec, secret: str | None) -> str:
|
|
502
458
|
if secret is not None:
|
|
503
459
|
return secret
|
|
504
|
-
value = os.environ.get(spec.secret_env or "")
|
|
460
|
+
value = os.environ.get(spec.auth.secret_env or "")
|
|
505
461
|
if not value:
|
|
506
|
-
fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
|
|
462
|
+
fail("secret", f"Set {spec.auth.secret_env} to the connection's credential", "/auth/secret_env")
|
|
507
463
|
return value
|
|
508
464
|
|
|
509
465
|
|
|
510
466
|
def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
|
|
511
|
-
"""The value the requests carry: the credential itself, or
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
return mint_token(spec, secret=secret, opener=opener)
|
|
517
|
-
return _secret_value(spec, secret)
|
|
467
|
+
"""The value the requests carry: the credential itself, or the token the
|
|
468
|
+
scheme exchanges it for first (oauth2)."""
|
|
469
|
+
return spec.auth.scheme.capture_value(
|
|
470
|
+
lambda: _secret_value(spec, secret), lambda: mint_token(spec, secret=secret, opener=opener)
|
|
471
|
+
)
|
|
518
472
|
|
|
519
473
|
|
|
520
474
|
def build_request(
|
|
@@ -540,19 +494,14 @@ def build_request(
|
|
|
540
494
|
query = list(pairs_of(params))
|
|
541
495
|
recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
542
496
|
headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
|
|
543
|
-
for header, variable in spec.extra_headers.items():
|
|
497
|
+
for header, variable in spec.auth.extra_headers.items():
|
|
544
498
|
value = os.environ.get(variable)
|
|
545
499
|
if not value:
|
|
546
500
|
fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
|
|
547
501
|
headers[header] = value
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
headers["Authorization"] = f"Bearer {secret}"
|
|
552
|
-
elif spec.scheme == "basic":
|
|
553
|
-
headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
|
|
554
|
-
elif spec.scheme == "query":
|
|
555
|
-
query.append((spec.auth_name or "", secret or ""))
|
|
502
|
+
carried_headers, carried_query = spec.auth.scheme.carry(secret or "")
|
|
503
|
+
headers.update(carried_headers)
|
|
504
|
+
query.extend(carried_query.items())
|
|
556
505
|
url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
557
506
|
return urllib.request.Request(url, method=operation.method, headers=headers), recorded
|
|
558
507
|
|
|
@@ -567,29 +516,15 @@ def undeclared_params(spec: Spec, operation: Operation, params: Params) -> list[
|
|
|
567
516
|
return seen
|
|
568
517
|
|
|
569
518
|
|
|
570
|
-
def token_endpoint(spec: Spec) -> str:
|
|
571
|
-
"""Where an oauth2 connection exchanges its client credentials."""
|
|
572
|
-
url = spec.token_url or DEFAULT_TOKEN_PATH
|
|
573
|
-
return url if url.startswith(("https://", "http://")) else spec.base_url + url
|
|
574
|
-
|
|
575
|
-
|
|
576
519
|
def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
|
|
577
520
|
"""Exchange the client credentials for an access token (RFC 6749 §4.4).
|
|
578
521
|
|
|
579
522
|
The client id and secret come from the environment the way every other
|
|
580
523
|
credential does; the token is what the captures then send as a bearer.
|
|
581
524
|
Nothing about the exchange is recorded, so the secret never reaches disk."""
|
|
582
|
-
|
|
583
|
-
fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
|
|
584
|
-
client_id = os.environ.get(spec.client_id_env or "")
|
|
585
|
-
if not client_id:
|
|
586
|
-
fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
|
|
587
|
-
client_secret = _secret_value(spec, secret)
|
|
588
|
-
form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
|
|
589
|
-
if spec.scope:
|
|
590
|
-
form["scope"] = spec.scope
|
|
525
|
+
endpoint, form = spec.auth.scheme.token_request(spec.base_url, lambda: _secret_value(spec, secret), os.environ)
|
|
591
526
|
request = urllib.request.Request(
|
|
592
|
-
|
|
527
|
+
endpoint,
|
|
593
528
|
method="POST",
|
|
594
529
|
data=urllib.parse.urlencode(form).encode(),
|
|
595
530
|
headers={
|
|
@@ -808,7 +743,9 @@ def capture(
|
|
|
808
743
|
following = urllib.parse.urlsplit(cursor)
|
|
809
744
|
if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
|
|
810
745
|
fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
|
|
811
|
-
page_params = [
|
|
746
|
+
page_params = [
|
|
747
|
+
(k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != spec.auth.scheme.secret_param
|
|
748
|
+
]
|
|
812
749
|
elif cursor:
|
|
813
750
|
cursor_param = spec.cursor_param or ""
|
|
814
751
|
page_params = [(k, v) for k, v in page_params if k != cursor_param] + [(cursor_param, cursor)]
|
|
@@ -20,6 +20,7 @@ base_url = "https://your-domain.atlassian.net"
|
|
|
20
20
|
# UI's; a world served with no key accepts any non-empty one.
|
|
21
21
|
[auth]
|
|
22
22
|
scheme = "basic"
|
|
23
|
+
credential = "password"
|
|
23
24
|
secret_env = "CONFLUENCE_API_TOKEN"
|
|
24
25
|
|
|
25
26
|
[capture]
|
|
@@ -60,8 +61,6 @@ entry = "/"
|
|
|
60
61
|
api_path = "/__world-api"
|
|
61
62
|
generated = false
|
|
62
63
|
|
|
63
|
-
# ---------------------------------------------------------------- spaces
|
|
64
|
-
|
|
65
64
|
[[operations]]
|
|
66
65
|
name = "get_spaces"
|
|
67
66
|
method = "GET"
|
|
@@ -96,8 +95,6 @@ filter = { id = "spaceId", status = "status", title = "title" }
|
|
|
96
95
|
match = { title = "contains" }
|
|
97
96
|
sample = { id = "65537" }
|
|
98
97
|
|
|
99
|
-
# ---------------------------------------------------------------- pages
|
|
100
|
-
|
|
101
98
|
[[operations]]
|
|
102
99
|
name = "get_pages"
|
|
103
100
|
method = "GET"
|
|
@@ -193,8 +190,6 @@ results = "results"
|
|
|
193
190
|
filter = { id = "pageId" }
|
|
194
191
|
sample = { id = "100063" }
|
|
195
192
|
|
|
196
|
-
# ---------------------------------------------------------------- comments
|
|
197
|
-
|
|
198
193
|
[[operations]]
|
|
199
194
|
name = "get_page_footer_comments"
|
|
200
195
|
method = "GET"
|
|
@@ -265,8 +260,6 @@ handler = "update_inline_comment"
|
|
|
265
260
|
sample = { commentId = "cmt-005" }
|
|
266
261
|
sample_body = { version = { number = 2 }, resolved = true }
|
|
267
262
|
|
|
268
|
-
# ---------------------------------------------------------------- labels
|
|
269
|
-
|
|
270
263
|
[[operations]]
|
|
271
264
|
name = "get_labels"
|
|
272
265
|
method = "GET"
|
|
@@ -289,8 +282,6 @@ action = "handler"
|
|
|
289
282
|
handler = "get_page_labels"
|
|
290
283
|
sample = { id = "100063" }
|
|
291
284
|
|
|
292
|
-
# ---------------------------------------------------------------- the ui's own reads
|
|
293
|
-
#
|
|
294
285
|
# One route per call in the UI's api client. Every one is a handler: these are
|
|
295
286
|
# composed read models (a page with its ancestors, children, comments and versions;
|
|
296
287
|
# a space tree), not vendor routes, and no vendor document describes them.
|
|
@@ -417,8 +408,6 @@ results = ""
|
|
|
417
408
|
action = "handler"
|
|
418
409
|
handler = "ui_labels"
|
|
419
410
|
|
|
420
|
-
# ---------------------------------------------------------------- the ui's writes
|
|
421
|
-
|
|
422
411
|
[[operations]]
|
|
423
412
|
name = "ui_add_comment"
|
|
424
413
|
method = "POST"
|
|
@@ -44,6 +44,7 @@ base_url = "https://instance.service-now.com"
|
|
|
44
44
|
# is the other supported scheme and is registered per instance. No credential
|
|
45
45
|
# ships with the template; the served world accepts the host's session credential.
|
|
46
46
|
scheme = "basic"
|
|
47
|
+
credential = "password"
|
|
47
48
|
secret_env = "SERVICENOW_BASIC_AUTH"
|
|
48
49
|
|
|
49
50
|
[capture]
|
|
@@ -98,7 +99,7 @@ not_found_status = 404
|
|
|
98
99
|
invalid_status = 400
|
|
99
100
|
|
|
100
101
|
[ui]
|
|
101
|
-
#
|
|
102
|
+
# The Next Experience twin, built from twins/servicenow/ui with
|
|
102
103
|
# no source edits. `generated = false`: ui/static is a build artifact of that
|
|
103
104
|
# directory, not the generated page, so compile leaves it alone. The SPA calls the api at the same origin
|
|
104
105
|
# under /__world-api, which the World Host rewrites onto the routes below.
|
|
@@ -109,8 +110,6 @@ entry = "/"
|
|
|
109
110
|
api_path = "/__world-api"
|
|
110
111
|
generated = false
|
|
111
112
|
|
|
112
|
-
# ------------------------------------------------------- the Table API (vendor)
|
|
113
|
-
|
|
114
113
|
[[operations]]
|
|
115
114
|
name = "list_table_records"
|
|
116
115
|
path = "/api/now/table/{tableName}"
|
|
@@ -192,8 +191,6 @@ handler = "delete_table_record"
|
|
|
192
191
|
status = 204
|
|
193
192
|
sample = { tableName = "incident_task", sys_id = "eaaefd673662a9417c46de3a53e6b147" }
|
|
194
193
|
|
|
195
|
-
# ------------------------------- the Next Experience shell (what the ui calls)
|
|
196
|
-
|
|
197
194
|
[[operations]]
|
|
198
195
|
name = "get_me"
|
|
199
196
|
path = "/api/now/ui/me"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""How a world answers browsers: ``[api] cors = true`` says the vendor's API serves pages on any
|
|
2
|
+
origin, so a preflight is answered and every answer names any origin; without it a world sends
|
|
3
|
+
no CORS header and an OPTIONS request is routed like any other method."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
ALLOW_ORIGIN = "Access-Control-Allow-Origin"
|
|
11
|
+
REQUESTED_HEADERS = "access-control-request-headers"
|
|
12
|
+
HEADER_NAMES = re.compile(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+(?:[ \t]*,[ \t]*[!#$%&'*+.^_`|~0-9A-Za-z-]+)*")
|
|
13
|
+
MAX_REQUESTED_CHARS = 1024
|
|
14
|
+
|
|
15
|
+
Preflight = tuple[int, None, dict[str, str]]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Cors:
|
|
20
|
+
"""A vendor whose API serves no browser."""
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def of(declared: bool, methods: tuple[str, ...]) -> Cors:
|
|
24
|
+
"""The world's CORS: :class:`OpenCors` over ``methods`` when declared, else none."""
|
|
25
|
+
return OpenCors(methods) if declared else Cors()
|
|
26
|
+
|
|
27
|
+
def preflight(self, method: str, headers: dict[str, str]) -> Preflight | None:
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
def carried(self, extra: dict[str, str] | None) -> dict[str, str] | None:
|
|
31
|
+
return extra
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class OpenCors(Cors):
|
|
36
|
+
"""A vendor whose API answers pages on any origin, without credentials. A preflight is
|
|
37
|
+
answered before any credential is checked, since a browser sends none with it: it names the
|
|
38
|
+
methods the world serves and, when they are a list of header names, the headers the page
|
|
39
|
+
asked to send. It reads nothing and writes nothing. Every other answer exposes its headers
|
|
40
|
+
(``Link``, the page headers) to the page."""
|
|
41
|
+
|
|
42
|
+
methods: tuple[str, ...]
|
|
43
|
+
|
|
44
|
+
def preflight(self, method: str, headers: dict[str, str]) -> Preflight | None:
|
|
45
|
+
if method != "OPTIONS":
|
|
46
|
+
return None
|
|
47
|
+
allowed = {ALLOW_ORIGIN: "*", "Access-Control-Allow-Methods": ", ".join(self.methods)}
|
|
48
|
+
requested = headers.get(REQUESTED_HEADERS, "")
|
|
49
|
+
if len(requested) <= MAX_REQUESTED_CHARS and HEADER_NAMES.fullmatch(requested):
|
|
50
|
+
allowed["Access-Control-Allow-Headers"] = requested
|
|
51
|
+
return 204, None, allowed
|
|
52
|
+
|
|
53
|
+
def carried(self, extra: dict[str, str] | None) -> dict[str, str] | None:
|
|
54
|
+
return {**(extra or {}), ALLOW_ORIGIN: "*", "Access-Control-Expose-Headers": "*"}
|