@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -10,10 +10,11 @@ connection is described once::
|
|
|
10
10
|
base_url = "https://api.spycloud.io/investigations-v2"
|
|
11
11
|
|
|
12
12
|
[auth]
|
|
13
|
-
scheme = "header" # header | bearer | basic | query | none
|
|
13
|
+
scheme = "header" # header | bearer | basic | query | oauth2 | none
|
|
14
14
|
name = "x-api-key" # header or query parameter that carries the secret
|
|
15
15
|
secret_env = "SPYCLOUD_API_KEY"
|
|
16
16
|
headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" } # more credential headers, each named by an env var
|
|
17
|
+
# [auth.login] and [auth.downloads]: the vendor's own login and signed links (door.py)
|
|
17
18
|
|
|
18
19
|
[capture]
|
|
19
20
|
requests_per_minute = 30 # pacing between requests; providers ask us to be sparing
|
|
@@ -58,7 +59,6 @@ the standard library, so the credential-free bridge can load it.
|
|
|
58
59
|
|
|
59
60
|
from __future__ import annotations
|
|
60
61
|
|
|
61
|
-
import base64
|
|
62
62
|
import hashlib
|
|
63
63
|
import json
|
|
64
64
|
import os
|
|
@@ -75,27 +75,35 @@ from pathlib import Path
|
|
|
75
75
|
from typing import Any, Callable, Iterable
|
|
76
76
|
|
|
77
77
|
from .compiler import check_bundle, compile_bundle
|
|
78
|
-
from .
|
|
78
|
+
from .database import WorldDatabase
|
|
79
|
+
from .door import VendorAuth
|
|
80
|
+
from .primary_key import PrimaryKey
|
|
79
81
|
from .transforms import apply_transforms
|
|
80
|
-
from .validation import
|
|
82
|
+
from .validation import (
|
|
83
|
+
FIELD_NAME,
|
|
84
|
+
METHODS,
|
|
85
|
+
PLACEHOLDER,
|
|
86
|
+
ContractError,
|
|
87
|
+
fail,
|
|
88
|
+
names_of,
|
|
89
|
+
operation_name,
|
|
90
|
+
parse_toml,
|
|
91
|
+
path_pattern,
|
|
92
|
+
route_path,
|
|
93
|
+
string_of,
|
|
94
|
+
table_of,
|
|
95
|
+
)
|
|
81
96
|
|
|
82
97
|
SPEC = "connector.toml"
|
|
83
98
|
CAPTURES = "captures"
|
|
84
99
|
MANIFEST = "manifest.json"
|
|
85
|
-
SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
|
|
86
|
-
DEFAULT_TOKEN_PATH = "/oauth/token"
|
|
87
|
-
METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
88
100
|
SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
89
|
-
#:
|
|
90
|
-
#:
|
|
91
|
-
|
|
92
|
-
#:
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*(\[\])?$")
|
|
96
|
-
PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
97
|
-
#: Query parameters as sent: an ordered list of (name, value) pairs, so a repeated
|
|
98
|
-
#: key (``expand[]=a&expand[]=b``) survives; a dict is accepted wherever pairs are.
|
|
101
|
+
#: A declared query parameter as the vendor names it: dotted (``query.cond``,
|
|
102
|
+
#: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
|
|
103
|
+
#: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
|
|
104
|
+
#: match or filter on it.
|
|
105
|
+
PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
|
|
106
|
+
# Pairs, not a dict, so a repeated key survives.
|
|
99
107
|
Params = list[tuple[str, str]]
|
|
100
108
|
MAX_BODY = 8 * 1024 * 1024
|
|
101
109
|
RECORDED_HEADERS = ("content-type", "retry-after", "x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-reset")
|
|
@@ -128,13 +136,9 @@ class Operation:
|
|
|
128
136
|
results: str
|
|
129
137
|
drop: tuple[str, ...] = ()
|
|
130
138
|
explode: tuple[Explode, ...] = ()
|
|
131
|
-
|
|
132
|
-
#: statement only knows its account from the ``{id}`` it was fetched under (Brex
|
|
133
|
-
#: retrospective); the captured row never carries it, so ingest stamps it on.
|
|
139
|
+
# A captured row may lack the path argument it was fetched under, so ingest stamps it on.
|
|
134
140
|
carry_args: dict[str, str] = field(default_factory=dict)
|
|
135
|
-
|
|
136
|
-
#: is one of them and the one a capture sends. Slack's Web API takes a read as
|
|
137
|
-
#: GET or POST alike, and its clients post everything.
|
|
141
|
+
# Slack's Web API takes a read as GET or POST alike, and its clients post everything.
|
|
138
142
|
methods: tuple[str, ...] = ()
|
|
139
143
|
|
|
140
144
|
def __post_init__(self) -> None:
|
|
@@ -145,20 +149,21 @@ class Operation:
|
|
|
145
149
|
def placeholders(self) -> tuple[str, ...]:
|
|
146
150
|
return tuple(PLACEHOLDER.findall(self.path))
|
|
147
151
|
|
|
152
|
+
@property
|
|
153
|
+
def pattern(self) -> re.Pattern:
|
|
154
|
+
"""The route's path as a pattern over a request's path, each ``{arg}`` one segment."""
|
|
155
|
+
return path_pattern(self.path)
|
|
156
|
+
|
|
157
|
+
def path_args(self, path: str) -> dict[str, str]:
|
|
158
|
+
found = self.pattern.match(path)
|
|
159
|
+
return {k: urllib.parse.unquote(v) for k, v in found.groupdict().items()} if found else {}
|
|
160
|
+
|
|
148
161
|
|
|
149
162
|
@dataclass(frozen=True)
|
|
150
163
|
class Spec:
|
|
151
164
|
slug: str
|
|
152
165
|
base_url: str
|
|
153
|
-
|
|
154
|
-
auth_name: str | None
|
|
155
|
-
secret_env: str | None
|
|
156
|
-
extra_headers: dict[str, str] # header name -> env var holding its value
|
|
157
|
-
#: oauth2 only: where the client-credentials grant is exchanged (absolute, or a
|
|
158
|
-
#: path under base_url), the env var naming the client id, and the scope asked for.
|
|
159
|
-
token_url: str | None
|
|
160
|
-
client_id_env: str | None
|
|
161
|
-
scope: str | None
|
|
166
|
+
auth: VendorAuth
|
|
162
167
|
requests_per_minute: int
|
|
163
168
|
timeout_seconds: float
|
|
164
169
|
max_pages: int
|
|
@@ -167,10 +172,15 @@ class Spec:
|
|
|
167
172
|
cursor_header: str | None
|
|
168
173
|
hash_fields: frozenset[str]
|
|
169
174
|
drop_fields: frozenset[str]
|
|
170
|
-
round_fields: dict[str, int] = field(default_factory=dict)
|
|
171
|
-
preserve_fields: frozenset[str] = frozenset()
|
|
175
|
+
round_fields: dict[str, int] = field(default_factory=dict)
|
|
176
|
+
preserve_fields: frozenset[str] = frozenset()
|
|
172
177
|
operations: dict[str, Operation] = field(default_factory=dict)
|
|
173
178
|
|
|
179
|
+
@property
|
|
180
|
+
def scheme(self) -> str:
|
|
181
|
+
"""The ``[auth] scheme`` the connection declares."""
|
|
182
|
+
return self.auth.scheme.name
|
|
183
|
+
|
|
174
184
|
def operation(self, name: str) -> Operation:
|
|
175
185
|
if name not in self.operations:
|
|
176
186
|
fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
|
|
@@ -200,26 +210,6 @@ class Spec:
|
|
|
200
210
|
)
|
|
201
211
|
|
|
202
212
|
|
|
203
|
-
# ---------------------------------------------------------------- spec
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
def _table(value: Any, location: str) -> dict:
|
|
207
|
-
if not isinstance(value, dict):
|
|
208
|
-
fail("spec", "Expected a table", location)
|
|
209
|
-
return value
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
|
|
213
|
-
if key not in table:
|
|
214
|
-
if required:
|
|
215
|
-
fail("spec", f"Missing {key}", location)
|
|
216
|
-
return default
|
|
217
|
-
value = table[key]
|
|
218
|
-
if not isinstance(value, str) or not value.strip():
|
|
219
|
-
fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
|
|
220
|
-
return value.strip()
|
|
221
|
-
|
|
222
|
-
|
|
223
213
|
RedactTable = tuple[frozenset[str], frozenset[str], dict[str, int], frozenset[str]]
|
|
224
214
|
|
|
225
215
|
|
|
@@ -227,12 +217,12 @@ def parse_redact(value: Any, location: str) -> RedactTable:
|
|
|
227
217
|
"""A ``[redact]`` table as ``(hash, drop, round, preserve)`` — the one grammar for
|
|
228
218
|
the policy wherever it is declared (``connector.toml`` for a vendor world,
|
|
229
219
|
``gateway-env.toml`` for a tools-only schema world)."""
|
|
230
|
-
redact =
|
|
220
|
+
redact = table_of(value, location)
|
|
231
221
|
hash_fields = frozenset(_paths(redact, "hash", location))
|
|
232
222
|
drop_fields = frozenset(_paths(redact, "drop", location))
|
|
233
223
|
rounding = redact.get("round", {})
|
|
234
224
|
if not isinstance(rounding, dict) or not all(
|
|
235
|
-
isinstance(k, str) and
|
|
225
|
+
isinstance(k, str) and FIELD_NAME.match(k) and isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= 10
|
|
236
226
|
for k, v in rounding.items()
|
|
237
227
|
):
|
|
238
228
|
fail("spec", "redact.round maps numeric field names to the decimals kept (0-10)", f"{location}/round")
|
|
@@ -249,7 +239,7 @@ def _paths(table: dict, key: str, location: str) -> tuple[str, ...]:
|
|
|
249
239
|
catalog's ``assets`` count map keeps its ``email`` key while records lose theirs."""
|
|
250
240
|
value = table.get(key, [])
|
|
251
241
|
if not isinstance(value, list) or not all(
|
|
252
|
-
isinstance(v, str) and all(part == "*" or
|
|
242
|
+
isinstance(v, str) and all(part == "*" or FIELD_NAME.match(part) for part in v.split(".")) and v for v in value
|
|
253
243
|
):
|
|
254
244
|
fail("spec", f"{key} must be a list of field names or dotted paths", f"{location}/{key}")
|
|
255
245
|
return tuple(value)
|
|
@@ -271,15 +261,6 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
|
|
|
271
261
|
return verbs
|
|
272
262
|
|
|
273
263
|
|
|
274
|
-
def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD) -> tuple[str, ...]:
|
|
275
|
-
value = table.get(key, [])
|
|
276
|
-
if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
|
|
277
|
-
fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
|
|
278
|
-
if len(set(value)) != len(value):
|
|
279
|
-
fail("spec", f"{key} repeats a name", f"{location}/{key}")
|
|
280
|
-
return tuple(value)
|
|
281
|
-
|
|
282
|
-
|
|
283
264
|
def pairs_of(params: Any) -> Params:
|
|
284
265
|
"""Query parameters as an ordered list of ``(name, value)`` pairs.
|
|
285
266
|
|
|
@@ -326,81 +307,67 @@ def parse_spec(text: str) -> Spec:
|
|
|
326
307
|
|
|
327
308
|
def spec_from(raw: dict) -> Spec:
|
|
328
309
|
"""The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
|
|
329
|
-
connector =
|
|
330
|
-
slug =
|
|
310
|
+
connector = table_of(raw.get("connector"), "/connector")
|
|
311
|
+
slug = string_of(connector, "slug", "/connector")
|
|
331
312
|
if not SLUG.match(slug):
|
|
332
313
|
fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
|
|
333
|
-
base_url =
|
|
314
|
+
base_url = string_of(connector, "base_url", "/connector")
|
|
334
315
|
parsed = urllib.parse.urlsplit(base_url)
|
|
335
316
|
if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
|
|
336
317
|
fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
|
|
337
318
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
|
|
343
|
-
secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
|
|
344
|
-
if scheme == "none" and (auth_name or secret_env):
|
|
345
|
-
fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
|
|
346
|
-
token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
|
|
347
|
-
client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
|
|
348
|
-
scope = _string(auth, "scope", "/auth", required=False)
|
|
349
|
-
if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
|
|
350
|
-
fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
|
|
351
|
-
if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
|
|
352
|
-
fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
|
|
353
|
-
extra_headers = auth.get("headers", {})
|
|
354
|
-
if not isinstance(extra_headers, dict) or not all(
|
|
355
|
-
isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
|
|
356
|
-
):
|
|
357
|
-
fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
|
|
358
|
-
if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
|
|
359
|
-
fail("spec", "auth.headers repeats the secret header", "/auth/headers")
|
|
360
|
-
|
|
361
|
-
capture = _table(raw.get("capture", {}), "/capture")
|
|
362
|
-
cursor_param = _string(capture, "cursor_param", "/capture", required=False)
|
|
363
|
-
cursor_field = _string(capture, "cursor_field", "/capture", required=False)
|
|
364
|
-
cursor_header = _string(capture, "cursor_header", "/capture", required=False)
|
|
319
|
+
capture = table_of(raw.get("capture", {}), "/capture")
|
|
320
|
+
cursor_param = string_of(capture, "cursor_param", "/capture", required=False)
|
|
321
|
+
cursor_field = string_of(capture, "cursor_field", "/capture", required=False)
|
|
322
|
+
cursor_header = string_of(capture, "cursor_header", "/capture", required=False)
|
|
365
323
|
if cursor_field and cursor_header:
|
|
366
324
|
fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
|
|
367
325
|
if bool(cursor_param) != bool(cursor_field or cursor_header):
|
|
368
326
|
fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
|
|
369
327
|
|
|
370
328
|
hash_fields, drop_fields, round_fields, preserve_fields = parse_redact(raw.get("redact", {}), "/redact")
|
|
329
|
+
operations = _operations(raw.get("operations", []))
|
|
330
|
+
|
|
331
|
+
return Spec(
|
|
332
|
+
slug=slug,
|
|
333
|
+
base_url=base_url.rstrip("/"),
|
|
334
|
+
auth=VendorAuth.parse(raw.get("auth", {"scheme": "none"}), operations),
|
|
335
|
+
requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
|
|
336
|
+
timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
|
|
337
|
+
max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
|
|
338
|
+
cursor_param=cursor_param,
|
|
339
|
+
cursor_field=cursor_field,
|
|
340
|
+
cursor_header=cursor_header,
|
|
341
|
+
hash_fields=hash_fields,
|
|
342
|
+
drop_fields=drop_fields,
|
|
343
|
+
round_fields=round_fields,
|
|
344
|
+
preserve_fields=preserve_fields,
|
|
345
|
+
operations=operations,
|
|
346
|
+
)
|
|
347
|
+
|
|
371
348
|
|
|
349
|
+
def _operations(listed: Any) -> dict[str, Operation]:
|
|
350
|
+
"""Every ``[[operations]]`` entry by name; two that would answer the same request are refused."""
|
|
372
351
|
operations: dict[str, Operation] = {}
|
|
373
352
|
routes: dict[tuple[str, str, tuple[str, ...]], str] = {}
|
|
374
|
-
listed = raw.get("operations", [])
|
|
375
353
|
if not isinstance(listed, list) or not listed:
|
|
376
354
|
fail("spec", "Declare at least one [[operations]] entry", "/operations")
|
|
377
355
|
for index, entry in enumerate(listed):
|
|
378
356
|
location = f"/operations/{index}"
|
|
379
|
-
table =
|
|
380
|
-
name =
|
|
357
|
+
table = table_of(entry, location)
|
|
358
|
+
name = string_of(table, "name", location)
|
|
381
359
|
operation_name(name, f"{location}/name")
|
|
382
360
|
if name in operations:
|
|
383
361
|
fail("spec", f"operation {name} is declared twice", f"{location}/name")
|
|
384
362
|
methods = _verbs(table, location)
|
|
385
|
-
method =
|
|
363
|
+
method = string_of(table, "method", location, default=methods[0] if methods else "GET", required=False).upper()
|
|
386
364
|
if method not in METHODS:
|
|
387
365
|
fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
|
|
388
366
|
if methods and method not in methods:
|
|
389
367
|
fail("spec", f"method {method} must be one of methods ({', '.join(methods)})", f"{location}/method")
|
|
390
|
-
path =
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
placeholders = PLACEHOLDER.findall(path)
|
|
394
|
-
repeated = [p for p in placeholders if placeholders.count(p) > 1]
|
|
395
|
-
if repeated:
|
|
396
|
-
fail(
|
|
397
|
-
"spec",
|
|
398
|
-
f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
|
|
399
|
-
f"{location}/path",
|
|
400
|
-
argument=repeated[0],
|
|
401
|
-
)
|
|
402
|
-
params = _names(table, "params", location, PARAM)
|
|
403
|
-
entity = _string(table, "entity", location, required=False)
|
|
368
|
+
path = route_path(string_of(table, "path", location) or "", f"{location}/path")
|
|
369
|
+
params = names_of(table, "params", location, PARAM)
|
|
370
|
+
entity = string_of(table, "entity", location, required=False)
|
|
404
371
|
results = table.get("results", "")
|
|
405
372
|
if not isinstance(results, str):
|
|
406
373
|
fail("spec", "results must be a dotted path string", f"{location}/results")
|
|
@@ -427,29 +394,7 @@ def spec_from(raw: dict) -> Spec:
|
|
|
427
394
|
f"{location}/path",
|
|
428
395
|
)
|
|
429
396
|
routes[route] = name
|
|
430
|
-
|
|
431
|
-
return Spec(
|
|
432
|
-
slug=slug,
|
|
433
|
-
base_url=base_url.rstrip("/"),
|
|
434
|
-
scheme=scheme,
|
|
435
|
-
auth_name=auth_name,
|
|
436
|
-
secret_env=secret_env,
|
|
437
|
-
extra_headers=dict(extra_headers),
|
|
438
|
-
token_url=token_url if scheme == "oauth2" else None,
|
|
439
|
-
client_id_env=client_id_env if scheme == "oauth2" else None,
|
|
440
|
-
scope=scope if scheme == "oauth2" else None,
|
|
441
|
-
requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
|
|
442
|
-
timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
|
|
443
|
-
max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
|
|
444
|
-
cursor_param=cursor_param,
|
|
445
|
-
cursor_field=cursor_field,
|
|
446
|
-
cursor_header=cursor_header,
|
|
447
|
-
hash_fields=hash_fields,
|
|
448
|
-
drop_fields=drop_fields,
|
|
449
|
-
round_fields=round_fields,
|
|
450
|
-
preserve_fields=preserve_fields,
|
|
451
|
-
operations=operations,
|
|
452
|
-
)
|
|
397
|
+
return operations
|
|
453
398
|
|
|
454
399
|
|
|
455
400
|
def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
|
|
@@ -462,8 +407,8 @@ def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
|
|
|
462
407
|
|
|
463
408
|
|
|
464
409
|
def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...], dict[str, str]]:
|
|
465
|
-
table =
|
|
466
|
-
drop =
|
|
410
|
+
table = table_of(raw, location)
|
|
411
|
+
drop = names_of(table, "drop", location)
|
|
467
412
|
carry_args = _mapping(table, "carry_args", location)
|
|
468
413
|
explode: list[Explode] = []
|
|
469
414
|
listed = table.get("explode", [])
|
|
@@ -471,9 +416,9 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
|
|
|
471
416
|
fail("spec", "explode must be a list of tables", f"{location}/explode")
|
|
472
417
|
for index, entry in enumerate(listed):
|
|
473
418
|
where = f"{location}/explode/{index}"
|
|
474
|
-
item =
|
|
475
|
-
field_name =
|
|
476
|
-
entity =
|
|
419
|
+
item = table_of(entry, where)
|
|
420
|
+
field_name = string_of(item, "field", where)
|
|
421
|
+
entity = string_of(item, "entity", where)
|
|
477
422
|
carry = _mapping(item, "carry", where)
|
|
478
423
|
take = _mapping(item, "take", where)
|
|
479
424
|
if not carry and not take:
|
|
@@ -509,26 +454,21 @@ def load_spec(world: str | Path) -> Spec:
|
|
|
509
454
|
return bind_spec(spec, read_json(world_json))
|
|
510
455
|
|
|
511
456
|
|
|
512
|
-
# ---------------------------------------------------------------- requests
|
|
513
|
-
|
|
514
|
-
|
|
515
457
|
def _secret_value(spec: Spec, secret: str | None) -> str:
|
|
516
458
|
if secret is not None:
|
|
517
459
|
return secret
|
|
518
|
-
value = os.environ.get(spec.secret_env or "")
|
|
460
|
+
value = os.environ.get(spec.auth.secret_env or "")
|
|
519
461
|
if not value:
|
|
520
|
-
fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
|
|
462
|
+
fail("secret", f"Set {spec.auth.secret_env} to the connection's credential", "/auth/secret_env")
|
|
521
463
|
return value
|
|
522
464
|
|
|
523
465
|
|
|
524
466
|
def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
|
|
525
|
-
"""The value the requests carry: the credential itself, or
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
return mint_token(spec, secret=secret, opener=opener)
|
|
531
|
-
return _secret_value(spec, secret)
|
|
467
|
+
"""The value the requests carry: the credential itself, or the token the
|
|
468
|
+
scheme exchanges it for first (oauth2)."""
|
|
469
|
+
return spec.auth.scheme.capture_value(
|
|
470
|
+
lambda: _secret_value(spec, secret), lambda: mint_token(spec, secret=secret, opener=opener)
|
|
471
|
+
)
|
|
532
472
|
|
|
533
473
|
|
|
534
474
|
def build_request(
|
|
@@ -554,19 +494,14 @@ def build_request(
|
|
|
554
494
|
query = list(pairs_of(params))
|
|
555
495
|
recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
556
496
|
headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
|
|
557
|
-
for header, variable in spec.extra_headers.items():
|
|
497
|
+
for header, variable in spec.auth.extra_headers.items():
|
|
558
498
|
value = os.environ.get(variable)
|
|
559
499
|
if not value:
|
|
560
500
|
fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
|
|
561
501
|
headers[header] = value
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
headers["Authorization"] = f"Bearer {secret}"
|
|
566
|
-
elif spec.scheme == "basic":
|
|
567
|
-
headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
|
|
568
|
-
elif spec.scheme == "query":
|
|
569
|
-
query.append((spec.auth_name or "", secret or ""))
|
|
502
|
+
carried_headers, carried_query = spec.auth.scheme.carry(secret or "")
|
|
503
|
+
headers.update(carried_headers)
|
|
504
|
+
query.extend(carried_query.items())
|
|
570
505
|
url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
571
506
|
return urllib.request.Request(url, method=operation.method, headers=headers), recorded
|
|
572
507
|
|
|
@@ -581,29 +516,15 @@ def undeclared_params(spec: Spec, operation: Operation, params: Params) -> list[
|
|
|
581
516
|
return seen
|
|
582
517
|
|
|
583
518
|
|
|
584
|
-
def token_endpoint(spec: Spec) -> str:
|
|
585
|
-
"""Where an oauth2 connection exchanges its client credentials."""
|
|
586
|
-
url = spec.token_url or DEFAULT_TOKEN_PATH
|
|
587
|
-
return url if url.startswith(("https://", "http://")) else spec.base_url + url
|
|
588
|
-
|
|
589
|
-
|
|
590
519
|
def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
|
|
591
520
|
"""Exchange the client credentials for an access token (RFC 6749 §4.4).
|
|
592
521
|
|
|
593
522
|
The client id and secret come from the environment the way every other
|
|
594
523
|
credential does; the token is what the captures then send as a bearer.
|
|
595
524
|
Nothing about the exchange is recorded, so the secret never reaches disk."""
|
|
596
|
-
|
|
597
|
-
fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
|
|
598
|
-
client_id = os.environ.get(spec.client_id_env or "")
|
|
599
|
-
if not client_id:
|
|
600
|
-
fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
|
|
601
|
-
client_secret = _secret_value(spec, secret)
|
|
602
|
-
form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
|
|
603
|
-
if spec.scope:
|
|
604
|
-
form["scope"] = spec.scope
|
|
525
|
+
endpoint, form = spec.auth.scheme.token_request(spec.base_url, lambda: _secret_value(spec, secret), os.environ)
|
|
605
526
|
request = urllib.request.Request(
|
|
606
|
-
|
|
527
|
+
endpoint,
|
|
607
528
|
method="POST",
|
|
608
529
|
data=urllib.parse.urlencode(form).encode(),
|
|
609
530
|
headers={
|
|
@@ -633,9 +554,6 @@ def _open(request: urllib.request.Request, timeout: float) -> Response:
|
|
|
633
554
|
fail("transport", f"Could not reach the connection: {error}", "/capture")
|
|
634
555
|
|
|
635
556
|
|
|
636
|
-
# ---------------------------------------------------------------- redaction
|
|
637
|
-
|
|
638
|
-
|
|
639
557
|
def _named(key: str, here: str, names: frozenset[str]) -> bool:
|
|
640
558
|
return key in names or here in names
|
|
641
559
|
|
|
@@ -759,7 +677,6 @@ def redact(value: Any, spec: Spec, path: str = "") -> Any:
|
|
|
759
677
|
elif _named(key, here, spec.preserve_fields) and item is not None:
|
|
760
678
|
out[key] = _shaped(spec.slug, item)
|
|
761
679
|
elif key in spec.round_fields and isinstance(item, (int, float)) and not isinstance(item, bool):
|
|
762
|
-
# Coordinates and the like: keep the shape and the type, lose the precision.
|
|
763
680
|
out[key] = round(item, spec.round_fields[key])
|
|
764
681
|
else:
|
|
765
682
|
out[key] = redact(item, spec, here)
|
|
@@ -777,9 +694,6 @@ def _dig(body: Any, path: str) -> Any:
|
|
|
777
694
|
return body
|
|
778
695
|
|
|
779
696
|
|
|
780
|
-
# ---------------------------------------------------------------- capture
|
|
781
|
-
|
|
782
|
-
|
|
783
697
|
def capture(
|
|
784
698
|
world: str | Path,
|
|
785
699
|
op: str,
|
|
@@ -825,12 +739,13 @@ def capture(
|
|
|
825
739
|
sleep(pause)
|
|
826
740
|
page_params = list(params)
|
|
827
741
|
if cursor and cursor.startswith(("https://", "http://")):
|
|
828
|
-
#
|
|
829
|
-
# query, never its host — the page must come from the connection described.
|
|
742
|
+
# Follow a next-page URL's query, never its host: the page must come from the connection described.
|
|
830
743
|
following = urllib.parse.urlsplit(cursor)
|
|
831
744
|
if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
|
|
832
745
|
fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
|
|
833
|
-
page_params = [
|
|
746
|
+
page_params = [
|
|
747
|
+
(k, v) for k, v in urllib.parse.parse_qsl(following.query) if k != spec.auth.scheme.secret_param
|
|
748
|
+
]
|
|
834
749
|
elif cursor:
|
|
835
750
|
cursor_param = spec.cursor_param or ""
|
|
836
751
|
page_params = [(k, v) for k, v in page_params if k != cursor_param] + [(cursor_param, cursor)]
|
|
@@ -898,22 +813,12 @@ def _append_manifest(root: Path, path: Path, record: dict, sha256: str) -> None:
|
|
|
898
813
|
manifest_path.write_text(json.dumps(entries, indent=1) + "\n")
|
|
899
814
|
|
|
900
815
|
|
|
901
|
-
# ---------------------------------------------------------------- ingest
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
def _key(contract: Any, entity: str, row: dict) -> tuple | None:
|
|
905
|
-
fields = contract.entities.get(entity, {}).get("primary_key", [])
|
|
906
|
-
if not fields or any(field not in row for field in fields):
|
|
907
|
-
return None
|
|
908
|
-
return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
|
|
909
|
-
|
|
910
|
-
|
|
911
816
|
def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list[dict]:
|
|
912
817
|
"""Insert what is new, update what changed, skip what the world already holds."""
|
|
913
818
|
held: dict[tuple[str, tuple], dict] = {}
|
|
914
819
|
for mutation in prior:
|
|
915
820
|
if mutation.get("op") in ("insert", "update") and isinstance(mutation.get("value"), dict):
|
|
916
|
-
key =
|
|
821
|
+
key = PrimaryKey.in_contract(contract, mutation["entity"]).identity(mutation["value"])
|
|
917
822
|
if key is not None:
|
|
918
823
|
held[(mutation["entity"], key)] = mutation["value"]
|
|
919
824
|
mutations: list[dict] = []
|
|
@@ -926,7 +831,8 @@ def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list
|
|
|
926
831
|
if token in seen:
|
|
927
832
|
continue
|
|
928
833
|
seen[token] = entry["file"]
|
|
929
|
-
|
|
834
|
+
primary = PrimaryKey.in_contract(contract, entity)
|
|
835
|
+
key = primary.identity(row)
|
|
930
836
|
existing = held.get((entity, key)) if key is not None else None
|
|
931
837
|
if existing == row:
|
|
932
838
|
continue
|
|
@@ -940,8 +846,7 @@ def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list
|
|
|
940
846
|
},
|
|
941
847
|
}
|
|
942
848
|
if existing is not None:
|
|
943
|
-
|
|
944
|
-
mutation["key"] = {field: row[field] for field in fields}
|
|
849
|
+
mutation["key"] = primary.key(row)
|
|
945
850
|
mutations.append(mutation)
|
|
946
851
|
return mutations
|
|
947
852
|
|
|
@@ -996,9 +901,7 @@ def response_rows(
|
|
|
996
901
|
for raw_row in rows:
|
|
997
902
|
if not isinstance(raw_row, dict):
|
|
998
903
|
fail("ingest", f"A row in {where} is not an object", pointer)
|
|
999
|
-
# The request is the authority for
|
|
1000
|
-
# names the field itself is overwritten. A parameter the capture was made
|
|
1001
|
-
# without is not stamped; the contract then says whether the row lacks it.
|
|
904
|
+
# The request is the authority for carried fields: a row naming the field itself is overwritten.
|
|
1002
905
|
carried = {name: context[source] for name, source in operation.carry_args.items() if source in context}
|
|
1003
906
|
out.extend(_project(operation, {**raw_row, **carried}, where, pointer))
|
|
1004
907
|
return out
|
|
@@ -1074,17 +977,16 @@ def _prove(root: Path, contract: Any, batch: dict) -> dict[str, int]:
|
|
|
1074
977
|
"""Apply the batch in a scratch copy of the world; the entity counts it leaves, or the
|
|
1075
978
|
store's refusal. The world on disk is never touched."""
|
|
1076
979
|
from .._materialize import materialize_template # only ingest needs the parent package
|
|
980
|
+
from .snapshot import build_names
|
|
1077
981
|
|
|
1078
982
|
with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
|
|
1079
983
|
staged = Path(scratch) / "world"
|
|
1080
|
-
shutil.copytree(
|
|
1081
|
-
root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", "state.sqlite", "state.json")
|
|
1082
|
-
)
|
|
984
|
+
shutil.copytree(root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", *build_names()))
|
|
1083
985
|
(staged / "data/initial.json").write_text(json.dumps(batch, sort_keys=True, indent=1) + "\n")
|
|
1084
986
|
compile_bundle(staged)
|
|
1085
987
|
database = Path(scratch) / "world.db"
|
|
1086
988
|
materialize_template(staged, database)
|
|
1087
|
-
with
|
|
989
|
+
with WorldDatabase.open(database, check_bundle(staged)) as store:
|
|
1088
990
|
store.audit()
|
|
1089
991
|
return {entity: len(store.list(entity)) for entity in contract.entities}
|
|
1090
992
|
|
|
@@ -1168,5 +1070,5 @@ def _counts(root: Path, contract: Any) -> dict[str, int]:
|
|
|
1168
1070
|
with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
|
|
1169
1071
|
database = Path(scratch) / "world.db"
|
|
1170
1072
|
materialize_template(root, database)
|
|
1171
|
-
with
|
|
1073
|
+
with WorldDatabase.open(database, contract) as store:
|
|
1172
1074
|
return {entity: len(store.list(entity)) for entity in contract.entities}
|
|
@@ -20,6 +20,7 @@ base_url = "https://your-domain.atlassian.net"
|
|
|
20
20
|
# UI's; a world served with no key accepts any non-empty one.
|
|
21
21
|
[auth]
|
|
22
22
|
scheme = "basic"
|
|
23
|
+
credential = "password"
|
|
23
24
|
secret_env = "CONFLUENCE_API_TOKEN"
|
|
24
25
|
|
|
25
26
|
[capture]
|
|
@@ -60,8 +61,6 @@ entry = "/"
|
|
|
60
61
|
api_path = "/__world-api"
|
|
61
62
|
generated = false
|
|
62
63
|
|
|
63
|
-
# ---------------------------------------------------------------- spaces
|
|
64
|
-
|
|
65
64
|
[[operations]]
|
|
66
65
|
name = "get_spaces"
|
|
67
66
|
method = "GET"
|
|
@@ -96,8 +95,6 @@ filter = { id = "spaceId", status = "status", title = "title" }
|
|
|
96
95
|
match = { title = "contains" }
|
|
97
96
|
sample = { id = "65537" }
|
|
98
97
|
|
|
99
|
-
# ---------------------------------------------------------------- pages
|
|
100
|
-
|
|
101
98
|
[[operations]]
|
|
102
99
|
name = "get_pages"
|
|
103
100
|
method = "GET"
|
|
@@ -193,8 +190,6 @@ results = "results"
|
|
|
193
190
|
filter = { id = "pageId" }
|
|
194
191
|
sample = { id = "100063" }
|
|
195
192
|
|
|
196
|
-
# ---------------------------------------------------------------- comments
|
|
197
|
-
|
|
198
193
|
[[operations]]
|
|
199
194
|
name = "get_page_footer_comments"
|
|
200
195
|
method = "GET"
|
|
@@ -265,8 +260,6 @@ handler = "update_inline_comment"
|
|
|
265
260
|
sample = { commentId = "cmt-005" }
|
|
266
261
|
sample_body = { version = { number = 2 }, resolved = true }
|
|
267
262
|
|
|
268
|
-
# ---------------------------------------------------------------- labels
|
|
269
|
-
|
|
270
263
|
[[operations]]
|
|
271
264
|
name = "get_labels"
|
|
272
265
|
method = "GET"
|
|
@@ -289,8 +282,6 @@ action = "handler"
|
|
|
289
282
|
handler = "get_page_labels"
|
|
290
283
|
sample = { id = "100063" }
|
|
291
284
|
|
|
292
|
-
# ---------------------------------------------------------------- the ui's own reads
|
|
293
|
-
#
|
|
294
285
|
# One route per call in the UI's api client. Every one is a handler: these are
|
|
295
286
|
# composed read models (a page with its ancestors, children, comments and versions;
|
|
296
287
|
# a space tree), not vendor routes, and no vendor document describes them.
|
|
@@ -417,8 +408,6 @@ results = ""
|
|
|
417
408
|
action = "handler"
|
|
418
409
|
handler = "ui_labels"
|
|
419
410
|
|
|
420
|
-
# ---------------------------------------------------------------- the ui's writes
|
|
421
|
-
|
|
422
411
|
[[operations]]
|
|
423
412
|
name = "ui_add_comment"
|
|
424
413
|
method = "POST"
|