@withgateway/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +3971 -2338
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3950 -2318
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +214 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +206 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
- package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1456 -564
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1433 -557
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +211 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +203 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +96 -21
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +96 -21
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +300 -123
- package/dist/worlds/index.d.ts +300 -123
- package/dist/worlds/index.js +1385 -508
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1377 -501
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -515
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +22 -14
- package/skills/connector-schema-authoring/connector-worlds.md +26 -10
- package/skills/connector-schema-authoring/world-schema.md +111 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +16 -16
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -9,11 +9,12 @@ one scaffold writer) takes the rendered files, validates and compiles the tree
|
|
|
9
9
|
and removes it whole when anything refuses, so a rejected import leaves no
|
|
10
10
|
directory behind. ``--plan-only`` stops at the plan.
|
|
11
11
|
|
|
12
|
-
Entities are the
|
|
13
|
-
|
|
14
|
-
whose operations name no schema gets every keyed
|
|
15
|
-
|
|
16
|
-
rules are in ``openapi_schemas``; routes in
|
|
12
|
+
Entities are the item schemas the selected operations answer and take, read
|
|
13
|
+
through the vendor's envelope (``openapi_payloads``), that a key is found for and
|
|
14
|
+
a planned route serves; a spec whose operations name no schema gets every keyed
|
|
15
|
+
schema instead. Every other schema stays a ``$defs`` entry referenced inline.
|
|
16
|
+
Keys, names and the shape rules are in ``openapi_schemas``; routes in
|
|
17
|
+
``openapi_routes``; paging in ``openapi_paging``.
|
|
17
18
|
|
|
18
19
|
Trust boundary: the source is read where the command runs (``openapi_source``:
|
|
19
20
|
a local file, or a public URL behind the private-address guard); a
|
|
@@ -30,13 +31,15 @@ import json
|
|
|
30
31
|
import re
|
|
31
32
|
from dataclasses import dataclass, field
|
|
32
33
|
from pathlib import Path
|
|
33
|
-
from typing import Any
|
|
34
|
+
from typing import Any, Iterable
|
|
34
35
|
|
|
35
36
|
from .compiler import Contract
|
|
36
37
|
from .connector import SPEC
|
|
37
38
|
from .openapi_document import OpenApiDocument, OpenApiOperation
|
|
39
|
+
from .openapi_paging import PagingPlan
|
|
40
|
+
from .openapi_payloads import Payloads, Sample
|
|
38
41
|
from .openapi_routes import Entity, OperationPlan, RoutePlanner, Selection, Skipped
|
|
39
|
-
from .openapi_schemas import EntityKey, Names, Namespace, SchemaConverter
|
|
42
|
+
from .openapi_schemas import EntityKey, Names, Namespace, SchemaConverter, key_parts, type_list
|
|
40
43
|
from .openapi_source import OpenApiSource
|
|
41
44
|
from .scaffold import init_bundle_from_files
|
|
42
45
|
from .validation import DIALECT, MAX_ENTITY_NAME, fail
|
|
@@ -62,8 +65,7 @@ class OpenApiWorldPlan:
|
|
|
62
65
|
excluded: int
|
|
63
66
|
defs: dict[str, dict]
|
|
64
67
|
warnings: list[str] = field(default_factory=list)
|
|
65
|
-
|
|
66
|
-
# ------------------------------------------------------------- the plan
|
|
68
|
+
paging: PagingPlan = field(default_factory=PagingPlan)
|
|
67
69
|
|
|
68
70
|
def summary(self) -> dict:
|
|
69
71
|
return {
|
|
@@ -77,8 +79,6 @@ class OpenApiWorldPlan:
|
|
|
77
79
|
"warnings": list(self.warnings),
|
|
78
80
|
}
|
|
79
81
|
|
|
80
|
-
# ------------------------------------------------------------- the files
|
|
81
|
-
|
|
82
82
|
def world(self) -> dict:
|
|
83
83
|
entities = {
|
|
84
84
|
e.name: {"schema": {"$ref": f"#/$defs/{_pointer(e.source)}"}, "primary_key": [e.key]}
|
|
@@ -107,10 +107,7 @@ class OpenApiWorldPlan:
|
|
|
107
107
|
"",
|
|
108
108
|
*self._auth_lines(),
|
|
109
109
|
"[api]",
|
|
110
|
-
|
|
111
|
-
'hits_key = ""',
|
|
112
|
-
"cursor_absent_when_done = true",
|
|
113
|
-
'page = "none"',
|
|
110
|
+
*self.paging.api_lines(),
|
|
114
111
|
"",
|
|
115
112
|
]
|
|
116
113
|
return "\n".join(head) + "\n" + "\n".join(o.toml() for o in self.operations)
|
|
@@ -154,8 +151,6 @@ class OpenApiWorldPlan:
|
|
|
154
151
|
PROVENANCE: _json_bytes(self.provenance()),
|
|
155
152
|
}
|
|
156
153
|
|
|
157
|
-
# ------------------------------------------------------------- writing
|
|
158
|
-
|
|
159
154
|
def write(self, destination: str | Path, *, name: str | None = None) -> Contract:
|
|
160
155
|
"""Hand the rendered files to the scaffold writer; nothing is written here."""
|
|
161
156
|
if not self.entities:
|
|
@@ -190,9 +185,7 @@ class OpenApiWorldPlan:
|
|
|
190
185
|
f"gateway worlds serve {destination} # then, in another shell:",
|
|
191
186
|
]
|
|
192
187
|
if listing is not None:
|
|
193
|
-
steps.append(
|
|
194
|
-
f"gateway worlds call http://127.0.0.1:<port> {listing.name} --args '{{}}' # GET {listing.path}"
|
|
195
|
-
)
|
|
188
|
+
steps.append(f"curl -s http://127.0.0.1:<port>{listing.path} # {listing.name}, over HTTP")
|
|
196
189
|
steps.append(
|
|
197
190
|
f'gateway worlds sandbox commit <sandbox-id> -m "import {self.source.label}" # when {destination} is in a sandbox'
|
|
198
191
|
)
|
|
@@ -238,13 +231,20 @@ class OpenApiImport:
|
|
|
238
231
|
if not selected:
|
|
239
232
|
fail("spec", "No operation matches --include/--exclude; nothing to import", "/paths")
|
|
240
233
|
converter = SchemaConverter(self.document, {})
|
|
241
|
-
|
|
242
|
-
|
|
234
|
+
payloads = Payloads(self.document, converter, selected).read()
|
|
235
|
+
every = not self._names_a_schema(selected)
|
|
236
|
+
entities = self._entities(converter, payloads, set(self.document.schemas) if every else payloads.items())
|
|
237
|
+
routes = RoutePlanner(self.document, entities, selected, payloads).plan()
|
|
238
|
+
entities = entities if every else self._routed(entities, routes, converter)
|
|
239
|
+
self._key_warnings(entities.values())
|
|
240
|
+
paging = PagingPlan({o.name: o.paging for o in routes.operations if o.paging is not None})
|
|
241
|
+
self._wrap(routes.operations, payloads, paging)
|
|
243
242
|
defs = converter.convert_all(set(entities))
|
|
244
243
|
self.warnings.extend(routes.warnings)
|
|
244
|
+
self.warnings.extend(paging.warnings())
|
|
245
245
|
self.warnings.extend(converter.warnings)
|
|
246
246
|
self.warnings.extend(
|
|
247
|
-
f"{at}: property {source} is stored as {legal} (not
|
|
247
|
+
f"{at}: property {source} is stored as {legal} (not a field name)"
|
|
248
248
|
for at, source, legal in converter.renamed
|
|
249
249
|
)
|
|
250
250
|
if converter.dropped:
|
|
@@ -260,30 +260,25 @@ class OpenApiImport:
|
|
|
260
260
|
excluded=len(everything) - len(selected),
|
|
261
261
|
defs=defs,
|
|
262
262
|
warnings=self.warnings,
|
|
263
|
+
paging=paging,
|
|
263
264
|
)
|
|
264
265
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
queue = [n for op in selected for n in self._refs(op.request_body, *op.responses.values()) if n is not None]
|
|
272
|
-
queue = [n for n in queue if n in self.document.schemas]
|
|
273
|
-
seen: set[str] = set()
|
|
274
|
-
while queue:
|
|
275
|
-
name = queue.pop()
|
|
276
|
-
if name in seen:
|
|
277
|
-
continue
|
|
278
|
-
seen.add(name)
|
|
279
|
-
for ref in _REF.findall(json.dumps(self.document.schemas[name])):
|
|
280
|
-
target = names(ref)
|
|
281
|
-
if target is not None and target in self.document.schemas and target not in seen:
|
|
282
|
-
queue.append(target)
|
|
283
|
-
if seen:
|
|
284
|
-
return seen
|
|
266
|
+
def _names_a_schema(self, selected: list[OpenApiOperation]) -> bool:
|
|
267
|
+
"""Whether some operation names a schema; a document whose operations name none
|
|
268
|
+
offers every keyed schema as an entity instead of the items its routes carry."""
|
|
269
|
+
named = [n for op in selected for n in self._refs(op.request_body, *op.responses.values())]
|
|
270
|
+
if any(n in self.document.schemas for n in named):
|
|
271
|
+
return True
|
|
285
272
|
self.warnings.append("no operation names a schema; every keyed schema becomes an entity")
|
|
286
|
-
return
|
|
273
|
+
return False
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def _routed(entities: dict[str, Entity], routes: RoutePlanner, converter: SchemaConverter) -> dict[str, Entity]:
|
|
277
|
+
"""The entities a planned route serves: one no route reads or writes would stay empty."""
|
|
278
|
+
used = {plan.entity for plan in routes.operations if plan.entity}
|
|
279
|
+
for source in [source for source, entity in entities.items() if entity.name not in used]:
|
|
280
|
+
del converter.roots[source]
|
|
281
|
+
return {source: entity for source, entity in entities.items() if entity.name in used}
|
|
287
282
|
|
|
288
283
|
def _refs(self, *nodes: Any) -> list[str | None]:
|
|
289
284
|
return [
|
|
@@ -293,49 +288,93 @@ class OpenApiImport:
|
|
|
293
288
|
for ref in _REF.findall(json.dumps(node))
|
|
294
289
|
]
|
|
295
290
|
|
|
296
|
-
def
|
|
297
|
-
"""
|
|
298
|
-
guessed for (first required scalar); error and request shapes never are."""
|
|
299
|
-
successes = [op.success[1] for op in selected if op.success is not None]
|
|
300
|
-
return {n for n in self._refs(*successes) if n is not None}
|
|
301
|
-
|
|
302
|
-
def _entities(self, converter: SchemaConverter, candidates: set[str], returned: set[str]) -> dict[str, Entity]:
|
|
303
|
-
"""The keyed object schemas among ``candidates``, registered as the converter's roots."""
|
|
291
|
+
def _entities(self, converter: SchemaConverter, payloads: Payloads, candidates: set[str]) -> dict[str, Entity]:
|
|
292
|
+
"""The candidates a key can be found for, registered as the converter's roots."""
|
|
304
293
|
names = Namespace(limit=MAX_ENTITY_NAME)
|
|
305
294
|
entities: dict[str, Entity] = {}
|
|
306
295
|
for source in sorted(candidates):
|
|
307
296
|
at = f"{self.document.schemas_pointer}/{_pointer(source)}"
|
|
308
|
-
|
|
309
|
-
if not
|
|
297
|
+
view = payloads.view({"$ref": f"#{at}"}, at)
|
|
298
|
+
if view is None or not view.properties:
|
|
310
299
|
continue
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
detector = EntityKey(self.document, source, properties, required)
|
|
317
|
-
key = detector.detect(fallback=source in returned)
|
|
300
|
+
detector = EntityKey(self.document, source, view.properties, view.required)
|
|
301
|
+
lookups = tuple(sorted(payloads.lookups.get(source, ())))
|
|
302
|
+
key = detector.detect(lookups=lookups, fallback=False)
|
|
303
|
+
guessed = key is None and source in payloads.lists
|
|
304
|
+
key = detector.detect(fallback=True) if guessed else key
|
|
318
305
|
if key is None:
|
|
319
306
|
continue
|
|
320
307
|
name, renamed = names.claim(Names.entity(source, MAX_ENTITY_NAME))
|
|
321
308
|
if renamed:
|
|
322
309
|
self.warnings.append(f"{at}: entity name for {source} is taken; using {name}")
|
|
323
|
-
fields = Names.fields(properties)
|
|
324
|
-
filterable = frozenset(fields[p] for p in detector.scalars())
|
|
310
|
+
fields = Names.fields(view.properties)
|
|
325
311
|
converter.roots[source] = key
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
)
|
|
312
|
+
naming = frozenset(own for own in lookups if key in (detector.top_level(own), detector.nested(own)))
|
|
313
|
+
entities[source] = self._entity(name, source, at, key, detector, fields, naming, guessed)
|
|
329
314
|
return entities
|
|
330
315
|
|
|
331
|
-
def
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
316
|
+
def _entity(
|
|
317
|
+
self,
|
|
318
|
+
name: str,
|
|
319
|
+
source: str,
|
|
320
|
+
at: str,
|
|
321
|
+
key: str,
|
|
322
|
+
detector: EntityKey,
|
|
323
|
+
fields: dict,
|
|
324
|
+
lookups: frozenset[str],
|
|
325
|
+
guessed: bool,
|
|
326
|
+
) -> Entity:
|
|
327
|
+
head, *rest = key_parts(key)
|
|
328
|
+
steps = [fields[head], *rest]
|
|
329
|
+
return Entity(
|
|
330
|
+
name,
|
|
331
|
+
source,
|
|
332
|
+
at,
|
|
333
|
+
"/" + "/".join(_pointer(step) for step in steps) if rest else fields[head],
|
|
334
|
+
self._key_type(detector.leaf(key)),
|
|
335
|
+
fields,
|
|
336
|
+
frozenset(fields[p] for p in detector.scalars()),
|
|
337
|
+
".".join(steps) if rest else "",
|
|
338
|
+
lookups,
|
|
339
|
+
guessed,
|
|
340
|
+
)
|
|
337
341
|
|
|
338
|
-
|
|
342
|
+
def _key_warnings(self, entities: Iterable[Entity]) -> None:
|
|
343
|
+
for entity in entities:
|
|
344
|
+
if entity.key_path:
|
|
345
|
+
self.warnings.append(
|
|
346
|
+
f"{entity.name} is keyed by {entity.key}, the field {{{', '.join(sorted(entity.lookups))}}} "
|
|
347
|
+
"reads a row by; every object on that path now requires its next step"
|
|
348
|
+
)
|
|
349
|
+
if entity.guessed:
|
|
350
|
+
self.warnings.append(
|
|
351
|
+
f"{entity.source} declares no key and no route reads one row of it: keyed by its first required "
|
|
352
|
+
f"field {entity.key}, so rows sharing it collapse; mark the key with x-primary-key, or include "
|
|
353
|
+
"the route that reads one row"
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
@staticmethod
|
|
357
|
+
def _key_type(resolved: dict) -> str:
|
|
358
|
+
return "integer" if [t for t in type_list(resolved.get("type")) if t != "null"] == ["integer"] else "string"
|
|
359
|
+
|
|
360
|
+
def _wrap(self, operations: list[OperationPlan], payloads: Payloads, paging: PagingPlan) -> None:
|
|
361
|
+
"""Each route's paging and, where the vendor wraps the rows, the rest of its envelope."""
|
|
362
|
+
sample = Sample(payloads)
|
|
363
|
+
for plan in operations:
|
|
364
|
+
plan.page = paging.route_page(plan.name)
|
|
365
|
+
if plan.entity and plan.payload is not None and plan.payload.path:
|
|
366
|
+
plan.wrapper = sample.envelope(plan.name, plan.payload, paging.filled)
|
|
367
|
+
if sample.unknown:
|
|
368
|
+
self.warnings.append(
|
|
369
|
+
f"the document gives no value for envelope fields {', '.join(sorted(sample.unknown))}: they are "
|
|
370
|
+
"served empty or 0; set the vendor's in [operations.api] wrapper"
|
|
371
|
+
)
|
|
372
|
+
if sample.cut:
|
|
373
|
+
self.warnings.append(
|
|
374
|
+
f"the envelopes of {', '.join(sorted(sample.cut))} pass the sampling budget ({Sample.FIELDS} fields or "
|
|
375
|
+
f"{Sample.ROUTE_BYTES // 1024} KiB a route, {Sample.TOTAL_BYTES // (1024 * 1024)} MiB in all): the rest "
|
|
376
|
+
"is left out of the served envelope; set it in [operations.api] wrapper"
|
|
377
|
+
)
|
|
339
378
|
|
|
340
379
|
|
|
341
380
|
def _request_source(request: dict) -> OpenApiSource:
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""How an imported world's list routes page, read from the document.
|
|
2
|
+
|
|
3
|
+
A list route pages when it declares the vendor's paging parameters and its envelope
|
|
4
|
+
carries where the next page starts: ClinicalTrials.gov's ``pageToken`` / ``pageSize``
|
|
5
|
+
with ``nextPageToken`` beside the rows, Crossref's ``cursor`` / ``rows`` with
|
|
6
|
+
``next-cursor``. A route declaring an ``offset`` pages by offset. The count beside the
|
|
7
|
+
rows (``totalCount``, ``total-results``) is the envelope's hits key either way. The
|
|
8
|
+
world's ``[api]`` table pages the way most list routes do; a list route that pages
|
|
9
|
+
otherwise, or not at all, says so on itself (``page = "cursor"``, ``page = "none"``). Paging by page number
|
|
10
|
+
(``page`` + ``per_page``) is not something the served world does, so it is named in a
|
|
11
|
+
warning and the route answers every row.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from functools import cached_property
|
|
18
|
+
|
|
19
|
+
from .openapi_document import OpenApiOperation
|
|
20
|
+
from .openapi_schemas import Names
|
|
21
|
+
|
|
22
|
+
LIMIT = ("limit", "per_page", "perPage", "page_size", "pageSize", "rows", "max_results", "maxResults")
|
|
23
|
+
OFFSET = ("offset",)
|
|
24
|
+
CURSOR = ("cursor", "page_token", "pageToken")
|
|
25
|
+
PAGE = ("page",)
|
|
26
|
+
#: Query parameters that page a list rather than select rows; never a filter.
|
|
27
|
+
PAGING_PARAMS = frozenset((*LIMIT, *OFFSET, *CURSOR, *PAGE))
|
|
28
|
+
_COUNTS = frozenset({"total", "count", "hits", "total_count", "total_results", "total_hits"})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class RoutePaging:
|
|
33
|
+
"""One list route's paging: the parameters it takes and the envelope keys it answers."""
|
|
34
|
+
|
|
35
|
+
style: str = "none"
|
|
36
|
+
limit_param: str | None = None
|
|
37
|
+
cursor_param: str | None = None
|
|
38
|
+
offset_param: str | None = None
|
|
39
|
+
page_size: int | None = None
|
|
40
|
+
cursor_key: str = ""
|
|
41
|
+
hits_key: str = ""
|
|
42
|
+
cursor_optional: bool = True
|
|
43
|
+
by_page_number: bool = False
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def of(cls, operation: OpenApiOperation, rows_at: dict | None) -> RoutePaging:
|
|
47
|
+
params = {p.name: p for p in operation.query_parameters}
|
|
48
|
+
siblings = rows_at.get("properties") if isinstance(rows_at, dict) else None
|
|
49
|
+
siblings = siblings if isinstance(siblings, dict) else {}
|
|
50
|
+
cursor_key = next((n for n, s in siblings.items() if _is_cursor(n, s)), "")
|
|
51
|
+
cursor = next((name for name in CURSOR if name in params), None)
|
|
52
|
+
offset = next((name for name in OFFSET if name in params), None)
|
|
53
|
+
limit = next((name for name in LIMIT if name in params), None)
|
|
54
|
+
styles = (("cursor", cursor is not None and bool(cursor_key)), ("offset", offset is not None))
|
|
55
|
+
style = next((name for name, applies in styles if applies), "none")
|
|
56
|
+
default = params[limit].schema.get("default") if limit is not None else None
|
|
57
|
+
return cls(
|
|
58
|
+
style=style,
|
|
59
|
+
limit_param=limit,
|
|
60
|
+
cursor_param=cursor if style == "cursor" else None,
|
|
61
|
+
offset_param=offset if style == "offset" else None,
|
|
62
|
+
page_size=default if isinstance(default, int) and not isinstance(default, bool) and default > 0 else None,
|
|
63
|
+
cursor_key=cursor_key if style == "cursor" else "",
|
|
64
|
+
hits_key=next((n for n, s in siblings.items() if _is_count(n, s)), ""),
|
|
65
|
+
cursor_optional=cursor_key not in (rows_at or {}).get("required", []),
|
|
66
|
+
by_page_number=style == "none" and any(name in params for name in PAGE),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class PagingPlan:
|
|
72
|
+
"""The world's ``[api]`` paging, from every list route's own: the style most of them
|
|
73
|
+
page by (a route paging otherwise names its own ``page``), and each parameter and
|
|
74
|
+
envelope key under the first name a route gives it."""
|
|
75
|
+
|
|
76
|
+
routes: dict[str, RoutePaging] = field(default_factory=dict)
|
|
77
|
+
|
|
78
|
+
@cached_property
|
|
79
|
+
def style(self) -> str:
|
|
80
|
+
styles = [paging.style for paging in self.routes.values() if paging.style != "none"]
|
|
81
|
+
return max(dict.fromkeys(styles), key=styles.count) if styles else "none"
|
|
82
|
+
|
|
83
|
+
def _first(self, attribute: str):
|
|
84
|
+
return next((value for paging in self.routes.values() if (value := getattr(paging, attribute))), None)
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def filled(self) -> frozenset[str]:
|
|
88
|
+
"""The envelope keys a served list writes itself: the count and the next cursor."""
|
|
89
|
+
return frozenset(key for key in (self._first("hits_key"), self._first("cursor_key")) if key)
|
|
90
|
+
|
|
91
|
+
def api_lines(self) -> list[str]:
|
|
92
|
+
cursor_key = self._first("cursor_key")
|
|
93
|
+
optional = next((p.cursor_optional for p in self.routes.values() if p.cursor_key), True)
|
|
94
|
+
lines = [
|
|
95
|
+
'envelope = ""',
|
|
96
|
+
f"hits_key = {_s(self._first('hits_key') or '')}",
|
|
97
|
+
*([f"cursor_key = {_s(cursor_key)}"] if cursor_key else []),
|
|
98
|
+
f"cursor_absent_when_done = {'true' if optional else 'false'}",
|
|
99
|
+
f"page = {_s(self.style)}",
|
|
100
|
+
]
|
|
101
|
+
if self.style == "none":
|
|
102
|
+
return lines
|
|
103
|
+
for key in ("limit_param", "offset_param", "cursor_param"):
|
|
104
|
+
value = self._first(key)
|
|
105
|
+
lines.extend([f"{key} = {_s(value)}"] if value else [])
|
|
106
|
+
size = self._first("page_size")
|
|
107
|
+
lines.extend([f"page_size = {size}"] if size else [])
|
|
108
|
+
return lines
|
|
109
|
+
|
|
110
|
+
def route_page(self, name: str) -> str | None:
|
|
111
|
+
"""The route's own ``page`` when it pages otherwise than the world; None when it does not."""
|
|
112
|
+
paging = self.routes.get(name)
|
|
113
|
+
return paging.style if paging is not None and paging.style != self.style else None
|
|
114
|
+
|
|
115
|
+
def warnings(self) -> list[str]:
|
|
116
|
+
out = []
|
|
117
|
+
for key in ("limit_param", "offset_param", "cursor_param"):
|
|
118
|
+
names = sorted({value for paging in self.routes.values() if (value := getattr(paging, key))})
|
|
119
|
+
if len(names) > 1:
|
|
120
|
+
out.append(
|
|
121
|
+
f"list routes name their {key} {', '.join(names)}; the world's [api] takes {self._first(key)}"
|
|
122
|
+
)
|
|
123
|
+
by_number = sorted(name for name, paging in self.routes.items() if paging.by_page_number)
|
|
124
|
+
if by_number:
|
|
125
|
+
out.append(
|
|
126
|
+
f"{', '.join(by_number)} page by page number, which a served world does not: one page holds every row"
|
|
127
|
+
)
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _is_cursor(name: str, schema: object) -> bool:
|
|
132
|
+
snake = Names.snake(name)
|
|
133
|
+
return _kind(schema) == "string" and ("cursor" in snake or snake.endswith("token"))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _is_count(name: str, schema: object) -> bool:
|
|
137
|
+
snake = Names.snake(name)
|
|
138
|
+
return _kind(schema) == "integer" and (snake in _COUNTS or snake.startswith("total_"))
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _kind(schema: object) -> str | None:
|
|
142
|
+
declared = schema.get("type") if isinstance(schema, dict) else None
|
|
143
|
+
return declared if isinstance(declared, str) else None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _s(text: str) -> str:
|
|
147
|
+
return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"'
|