@withgateway/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +3971 -2338
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3950 -2318
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +214 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +206 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
- package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1456 -564
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1433 -557
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +211 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +203 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +96 -21
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +96 -21
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +300 -123
- package/dist/worlds/index.d.ts +300 -123
- package/dist/worlds/index.js +1385 -508
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1377 -501
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -515
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +22 -14
- package/skills/connector-schema-authoring/connector-worlds.md +26 -10
- package/skills/connector-schema-authoring/world-schema.md +111 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +16 -16
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""What an OpenAPI response (or request body) carries: rows of one item schema,
|
|
2
|
+
many or one, and where they sit inside the vendor's envelope.
|
|
3
|
+
|
|
4
|
+
A vendor rarely answers the item bare. Crossref answers ``GET /works`` with
|
|
5
|
+
``{status, message-type, message: {items: [Work], total-results}}`` and
|
|
6
|
+
``GET /works/{doi}`` with ``{status, message-type, message: Work}``; ClinicalTrials.gov
|
|
7
|
+
answers ``{studies: [Study], nextPageToken}``. The wrappers are not entities — keyed by
|
|
8
|
+
their first required field they collapse every row into one — so the importer reads
|
|
9
|
+
through them to the item, and the served world answers the same envelope again.
|
|
10
|
+
|
|
11
|
+
:class:`Payloads` reads every selected operation in two passes. Lists first: an array
|
|
12
|
+
of objects, or an object that declares no key and holds exactly one array of objects,
|
|
13
|
+
possibly behind objects that hold nothing else (``message`` → ``items``). Then single
|
|
14
|
+
rows: the schema itself when it is a list's item, declares a key, or names the field
|
|
15
|
+
the route's own ``{segment}`` looks it up by; else, through a wrapper holding exactly
|
|
16
|
+
one object, that object when it is a list's item, declares a key or names the segment's
|
|
17
|
+
field; else the schema itself when the segment names a field nested inside it.
|
|
18
|
+
Anything else is no rows at all.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import json
|
|
24
|
+
import re
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
from .openapi_document import OpenApiDocument, OpenApiOperation
|
|
29
|
+
from .openapi_schemas import EntityKey, Names, SchemaConverter, type_list
|
|
30
|
+
|
|
31
|
+
MAX_DEPTH = 3
|
|
32
|
+
_LITERAL = re.compile(r"^[A-Za-z0-9 _.,:/-]+$")
|
|
33
|
+
_ABSENT = object()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class View:
|
|
38
|
+
"""A schema node read as an object: the top-level schema it names (if any) and
|
|
39
|
+
its properties after ``$ref`` and ``allOf`` are folded."""
|
|
40
|
+
|
|
41
|
+
name: str | None
|
|
42
|
+
schema: dict
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def properties(self) -> dict:
|
|
46
|
+
properties = self.schema.get("properties")
|
|
47
|
+
return properties if isinstance(properties, dict) else {}
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def required(self) -> set[str]:
|
|
51
|
+
return {r for r in self.schema.get("required", []) if isinstance(r, str)}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class Payload:
|
|
56
|
+
"""Rows of ``item``: ``many`` or one, at ``path`` inside the wrappers that hold them."""
|
|
57
|
+
|
|
58
|
+
item: str
|
|
59
|
+
many: bool
|
|
60
|
+
path: tuple[str, ...] = ()
|
|
61
|
+
wrappers: tuple[dict, ...] = ()
|
|
62
|
+
|
|
63
|
+
def under(self, key: str, wrapper: dict) -> Payload:
|
|
64
|
+
return Payload(self.item, self.many, (key, *self.path), (wrapper, *self.wrappers))
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def envelope(self) -> str:
|
|
68
|
+
return ".".join(self.path)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class Payloads:
|
|
73
|
+
"""Every selected operation's response and body read as rows of an item schema."""
|
|
74
|
+
|
|
75
|
+
document: OpenApiDocument
|
|
76
|
+
converter: SchemaConverter
|
|
77
|
+
selected: list[OpenApiOperation]
|
|
78
|
+
lists: set[str] = field(default_factory=set)
|
|
79
|
+
responses: dict[str, Payload | None] = field(default_factory=dict)
|
|
80
|
+
bodies: dict[str, Payload | None] = field(default_factory=dict)
|
|
81
|
+
lookups: dict[str, set[str]] = field(default_factory=dict)
|
|
82
|
+
|
|
83
|
+
def read(self) -> Payloads:
|
|
84
|
+
listed = {op.pointer: self._list(op.success[1], self._at(op), 0) for op in self.selected if op.success}
|
|
85
|
+
self.lists = {payload.item for payload in listed.values() if payload is not None}
|
|
86
|
+
for op in self.selected:
|
|
87
|
+
own = own_segment(op.path)
|
|
88
|
+
found = listed.get(op.pointer) or self._single(op.success[1] if op.success else None, self._at(op), own, 0)
|
|
89
|
+
self.responses[op.pointer] = found
|
|
90
|
+
self.bodies[op.pointer] = self._single(op.request_body, f"{op.pointer}/requestBody", None, 0)
|
|
91
|
+
if found is not None and not found.many and own is not None and op.method == "GET":
|
|
92
|
+
self.lookups.setdefault(found.item, set()).add(own)
|
|
93
|
+
return self
|
|
94
|
+
|
|
95
|
+
def items(self) -> set[str]:
|
|
96
|
+
return {p.item for p in (*self.responses.values(), *self.bodies.values()) if p is not None}
|
|
97
|
+
|
|
98
|
+
@staticmethod
|
|
99
|
+
def _at(op: OpenApiOperation) -> str:
|
|
100
|
+
return f"{op.pointer}/responses/{op.success[0]}" if op.success else op.pointer
|
|
101
|
+
|
|
102
|
+
def view(self, node: Any, at: str) -> View | None:
|
|
103
|
+
if not isinstance(node, dict):
|
|
104
|
+
return None
|
|
105
|
+
resolved, chain = self.converter.resolve_named(node, at, ())
|
|
106
|
+
if not isinstance(resolved, dict):
|
|
107
|
+
return None
|
|
108
|
+
merged = self.converter.merged(resolved, at, chain) if chain else self.converter.merge(resolved, at, ())
|
|
109
|
+
return View(chain[-1] if chain else None, merged)
|
|
110
|
+
|
|
111
|
+
def _object_item(self, node: Any, at: str) -> View | None:
|
|
112
|
+
item = self.view(node, at)
|
|
113
|
+
return item if item is not None and item.name is not None and item.properties else None
|
|
114
|
+
|
|
115
|
+
def _array_items(self, node: Any, at: str) -> View | None:
|
|
116
|
+
view = self.view(node, at)
|
|
117
|
+
if view is None or (view.schema.get("type") != "array" and "items" not in view.schema):
|
|
118
|
+
return None
|
|
119
|
+
return self._object_item(view.schema.get("items"), f"{at}/items")
|
|
120
|
+
|
|
121
|
+
def _object_members(self, view: View, at: str) -> list[tuple[str, View]]:
|
|
122
|
+
members = [(key, self.view(value, f"{at}/properties/{key}")) for key, value in view.properties.items()]
|
|
123
|
+
return [(key, member) for key, member in members if member is not None and member.properties]
|
|
124
|
+
|
|
125
|
+
def _array_members(self, view: View, at: str) -> list[str]:
|
|
126
|
+
return [key for key, value in view.properties.items() if self._array_items(value, f"{at}/properties/{key}")]
|
|
127
|
+
|
|
128
|
+
def keyed(self, view: View) -> bool:
|
|
129
|
+
if view.name is None:
|
|
130
|
+
return False
|
|
131
|
+
return EntityKey(self.document, view.name, view.properties, view.required).declared() is not None
|
|
132
|
+
|
|
133
|
+
def _list(self, node: Any, at: str, depth: int) -> Payload | None:
|
|
134
|
+
items = self._array_items(node, at)
|
|
135
|
+
if items is not None:
|
|
136
|
+
return Payload(items.name or "", True)
|
|
137
|
+
view = self.view(node, at)
|
|
138
|
+
if view is None or not view.properties or self.keyed(view) or depth >= MAX_DEPTH:
|
|
139
|
+
return None
|
|
140
|
+
arrays = self._array_members(view, at)
|
|
141
|
+
if len(arrays) == 1:
|
|
142
|
+
inner = self._list(view.properties[arrays[0]], f"{at}/properties/{arrays[0]}", depth + 1)
|
|
143
|
+
return None if inner is None else inner.under(arrays[0], view.schema)
|
|
144
|
+
objects = self._object_members(view, at)
|
|
145
|
+
if arrays or len(objects) != 1:
|
|
146
|
+
return None
|
|
147
|
+
key, _ = objects[0]
|
|
148
|
+
inner = self._list(view.properties[key], f"{at}/properties/{key}", depth + 1)
|
|
149
|
+
return None if inner is None else inner.under(key, view.schema)
|
|
150
|
+
|
|
151
|
+
def _single(self, node: Any, at: str, own: str | None, depth: int) -> Payload | None:
|
|
152
|
+
view = self._object_item(node, at)
|
|
153
|
+
if view is None:
|
|
154
|
+
return None
|
|
155
|
+
if view.name in self.lists or self.keyed(view) or self._names_field(view, own):
|
|
156
|
+
return Payload(view.name, False)
|
|
157
|
+
wrapped = self._wrapped(view, at, own, depth)
|
|
158
|
+
if wrapped is not None:
|
|
159
|
+
return wrapped
|
|
160
|
+
key = EntityKey(self.document, view.name, view.properties, view.required)
|
|
161
|
+
return Payload(view.name, False) if own is not None and key.nested(own) is not None else None
|
|
162
|
+
|
|
163
|
+
def _wrapped(self, view: View, at: str, own: str | None, depth: int) -> Payload | None:
|
|
164
|
+
objects = self._object_members(view, at)
|
|
165
|
+
if depth >= MAX_DEPTH or len(objects) != 1 or self._array_members(view, at):
|
|
166
|
+
return None
|
|
167
|
+
key, member = objects[0]
|
|
168
|
+
if member.name is None:
|
|
169
|
+
return None
|
|
170
|
+
if member.name in self.lists or self.keyed(member) or self._names_field(member, own):
|
|
171
|
+
return Payload(member.name, False, (key,), (view.schema,))
|
|
172
|
+
inner = self._wrapped(member, f"{at}/properties/{key}", own, depth + 1)
|
|
173
|
+
return None if inner is None else inner.under(key, view.schema)
|
|
174
|
+
|
|
175
|
+
@staticmethod
|
|
176
|
+
def _names_field(view: View, own: str | None) -> bool:
|
|
177
|
+
return own is not None and any(Names.snake(name) == Names.snake(own) for name in view.properties)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def own_segment(path: str) -> str | None:
|
|
181
|
+
"""The ``{segment}`` a path ends in: the one that names its own row (``/works/{doi}``),
|
|
182
|
+
as opposed to a scope earlier in the path (``/funders/{id}/works``)."""
|
|
183
|
+
last = path.rstrip("/").rsplit("/", 1)[-1]
|
|
184
|
+
match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", last)
|
|
185
|
+
return match.group(1) if match else None
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class Sample:
|
|
189
|
+
"""The smallest value a schema accepts, for the parts of a vendor envelope the
|
|
190
|
+
world has no data for (``status``, ``message-type``): the schema's own ``enum``,
|
|
191
|
+
``const``, ``default`` or ``example`` when it is a short scalar, a literal
|
|
192
|
+
``pattern``, else the type's zero. Budgets bound what an envelope can grow to,
|
|
193
|
+
however a document shares its schemas: one route's envelope samples at most
|
|
194
|
+
``FIELDS`` fields and ``ROUTE_BYTES`` of names and values, and all routes together
|
|
195
|
+
at most ``TOTAL_BYTES``; past a budget the rest is left out and the route is named
|
|
196
|
+
in ``cut``."""
|
|
197
|
+
|
|
198
|
+
ZEROS = {"string": "", "integer": 0, "number": 0, "boolean": False, "null": None}
|
|
199
|
+
FIELDS = 256
|
|
200
|
+
LONGEST = 256
|
|
201
|
+
ROUTE_BYTES = 16 * 1024
|
|
202
|
+
TOTAL_BYTES = 1024 * 1024
|
|
203
|
+
|
|
204
|
+
def __init__(self, payloads: Payloads):
|
|
205
|
+
self.payloads = payloads
|
|
206
|
+
self.unknown: set[str] = set()
|
|
207
|
+
self.cut: set[str] = set()
|
|
208
|
+
self._route = ""
|
|
209
|
+
self._fields = self.FIELDS
|
|
210
|
+
self._bytes = 0
|
|
211
|
+
self._total = self.TOTAL_BYTES
|
|
212
|
+
|
|
213
|
+
def envelope(self, route: str, payload: Payload, filled: frozenset[str]) -> dict:
|
|
214
|
+
"""The vendor envelope around ``payload``'s rows on ``route``, minus the rows
|
|
215
|
+
themselves and the ``filled`` keys the served route writes (the count and the
|
|
216
|
+
next cursor)."""
|
|
217
|
+
self._route, self._fields = route, self.FIELDS
|
|
218
|
+
self._bytes = budget = min(self.ROUTE_BYTES, self._total)
|
|
219
|
+
template: dict = {}
|
|
220
|
+
level = template
|
|
221
|
+
for depth, (key, wrapper) in enumerate(zip(payload.path, payload.wrappers)):
|
|
222
|
+
innermost = depth == len(payload.path) - 1
|
|
223
|
+
skip = frozenset({key}) | (filled if innermost else frozenset())
|
|
224
|
+
level.update(self.required_of(View(None, wrapper), "/envelope", skip=skip))
|
|
225
|
+
if innermost:
|
|
226
|
+
break
|
|
227
|
+
level = level.setdefault(key, {})
|
|
228
|
+
self._total -= budget - max(self._bytes, 0)
|
|
229
|
+
return template
|
|
230
|
+
|
|
231
|
+
def of(self, node: Any, at: str, name: str, depth: int = 0) -> Any:
|
|
232
|
+
view = self.payloads.view(node, at) if depth < MAX_DEPTH + 2 else None
|
|
233
|
+
schema = view.schema if view is not None else {}
|
|
234
|
+
given = [schema[key] for key in ("const", "default", "example") if key in schema]
|
|
235
|
+
enum = schema.get("enum")
|
|
236
|
+
given += [enum[0]] if isinstance(enum, list) and enum else []
|
|
237
|
+
short = next((value for value in given if self._short(value)), _ABSENT)
|
|
238
|
+
if short is not _ABSENT:
|
|
239
|
+
return short
|
|
240
|
+
kind = self._kind(schema)
|
|
241
|
+
if kind == "object":
|
|
242
|
+
return self.required_of(view, at, depth + 1) if view is not None else {}
|
|
243
|
+
if kind == "array":
|
|
244
|
+
return []
|
|
245
|
+
pattern = schema.get("pattern")
|
|
246
|
+
if kind == "string" and self._short(pattern) and isinstance(pattern, str) and _LITERAL.match(pattern):
|
|
247
|
+
return pattern
|
|
248
|
+
if kind in ("string", "integer", "number") and "minimum" not in schema:
|
|
249
|
+
self.unknown.add(name)
|
|
250
|
+
if (
|
|
251
|
+
kind in ("integer", "number")
|
|
252
|
+
and isinstance(schema.get("minimum"), (int, float))
|
|
253
|
+
and self._short(schema["minimum"])
|
|
254
|
+
):
|
|
255
|
+
return schema["minimum"]
|
|
256
|
+
return self.ZEROS.get(kind)
|
|
257
|
+
|
|
258
|
+
def required_of(self, view: View, at: str, depth: int = 0, skip: frozenset[str] = frozenset()) -> dict:
|
|
259
|
+
out: dict = {}
|
|
260
|
+
for key, value in view.properties.items():
|
|
261
|
+
if key not in view.required or key in skip:
|
|
262
|
+
continue
|
|
263
|
+
if self._fields <= 0 or self._bytes <= 0 or self._bytes < len(key):
|
|
264
|
+
self.cut.add(self._route)
|
|
265
|
+
return out
|
|
266
|
+
self._fields -= 1
|
|
267
|
+
self._bytes -= len(key)
|
|
268
|
+
out[key] = self.of(value, f"{at}/properties/{key}", key, depth)
|
|
269
|
+
self._bytes -= 2 if isinstance(out[key], dict) else len(json.dumps(out[key], ensure_ascii=False))
|
|
270
|
+
return out
|
|
271
|
+
|
|
272
|
+
@classmethod
|
|
273
|
+
def _short(cls, value: Any) -> bool:
|
|
274
|
+
"""A scalar a template may copy: a string of at most ``LONGEST`` characters, an
|
|
275
|
+
integer of at most ``LONGEST * 3`` bits (about 230 digits, judged without printing
|
|
276
|
+
it), a float, a boolean or null; never an object or an array."""
|
|
277
|
+
if isinstance(value, str):
|
|
278
|
+
return len(value) <= cls.LONGEST
|
|
279
|
+
if value is None or isinstance(value, (bool, float)):
|
|
280
|
+
return True
|
|
281
|
+
return isinstance(value, int) and value.bit_length() <= cls.LONGEST * 3
|
|
282
|
+
|
|
283
|
+
@staticmethod
|
|
284
|
+
def _kind(schema: dict) -> str | None:
|
|
285
|
+
kinds = [kind for kind in type_list(schema.get("type")) if kind != "null"]
|
|
286
|
+
if kinds:
|
|
287
|
+
return kinds[0]
|
|
288
|
+
return "object" if isinstance(schema.get("properties"), dict) else None
|
|
@@ -1,28 +1,31 @@
|
|
|
1
1
|
"""The routes an OpenAPI import plans: every path+method of the document as one
|
|
2
2
|
``[[operations]]`` entry of connector.toml, or a skip with its reason.
|
|
3
3
|
|
|
4
|
-
An operation gets an entity when its success response
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
4
|
+
An operation gets an entity when its success response carries rows of one
|
|
5
|
+
(:mod:`openapi_payloads`: the item itself, a list of it, or either inside the vendor's
|
|
6
|
+
envelope, whose path becomes the route's ``envelope``). Writes whose response says
|
|
7
|
+
nothing (``DELETE`` answering 204, Swagger's form updates) take the entity of the
|
|
8
|
+
``GET`` on the same path, the way a REST resource reads. Everything else stays a
|
|
9
|
+
*passthrough* operation: declared, served (``501 not_modelled``, so ``conform`` still
|
|
10
|
+
sees the route), carrying no entity, with the reason in the plan. Only what a
|
|
11
|
+
connector cannot declare at all is skipped: methods outside the allowed set,
|
|
12
|
+
placeholders that are not identifiers, and a route another operation already answers.
|
|
13
|
+
|
|
14
|
+
Path arguments bind the way ``route_bindings`` demands (W2): the path's own
|
|
15
|
+
``{segment}`` (the last one) names the row by its key (``petId``, ``doi`` for a
|
|
16
|
+
``DOI`` key, ``nctId`` for a key nested in the row) or by a field; an earlier
|
|
17
|
+
segment selects rows only when it names a field, else it is declared in ``params``
|
|
18
|
+
as a scope segment. A single read whose segments select nothing is a passthrough:
|
|
19
|
+
served as some row, it would answer any value with the wrong one. Declared query
|
|
20
|
+
parameters naming an entity field filter list routes (W5) and are said so in
|
|
21
|
+
``filter`` explicitly, so the plan and the served route agree without anyone
|
|
22
|
+
reading defaults.
|
|
21
23
|
"""
|
|
22
24
|
|
|
23
25
|
from __future__ import annotations
|
|
24
26
|
|
|
25
27
|
import fnmatch
|
|
28
|
+
import json
|
|
26
29
|
import re
|
|
27
30
|
from collections.abc import Callable
|
|
28
31
|
from dataclasses import dataclass, field
|
|
@@ -30,19 +33,19 @@ from typing import Any
|
|
|
30
33
|
|
|
31
34
|
from .connector import PARAM, PLACEHOLDER
|
|
32
35
|
from .openapi_document import OpenApiDocument, OpenApiOperation
|
|
36
|
+
from .openapi_paging import PAGING_PARAMS, RoutePaging
|
|
37
|
+
from .openapi_payloads import Payload, Payloads, own_segment
|
|
33
38
|
from .openapi_schemas import Names, Namespace
|
|
34
39
|
from .validation import MAX_OPERATION_NAME
|
|
35
40
|
|
|
36
|
-
#: Query parameters that page a list rather than select rows; never a filter.
|
|
37
|
-
PAGING_PARAMS = frozenset(
|
|
38
|
-
{"limit", "offset", "cursor", "page", "per_page", "perPage", "page_size", "pageSize", "page_token", "pageToken"}
|
|
39
|
-
)
|
|
40
41
|
_BRACED = re.compile(r"\{([^{}]*)\}")
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
@dataclass(frozen=True)
|
|
44
45
|
class Entity:
|
|
45
|
-
"""One planned entity: the schema it came from and the names it got.
|
|
46
|
+
"""One planned entity: the schema it came from and the names it got. ``key`` is a
|
|
47
|
+
field, or a JSON Pointer when the key is nested; ``key_path`` is the dotted path a
|
|
48
|
+
served filter reads it at; ``lookups`` are the path arguments that name the key."""
|
|
46
49
|
|
|
47
50
|
name: str
|
|
48
51
|
source: str
|
|
@@ -51,82 +54,51 @@ class Entity:
|
|
|
51
54
|
key_type: str
|
|
52
55
|
fields: dict[str, str]
|
|
53
56
|
filterable: frozenset[str] = frozenset()
|
|
57
|
+
key_path: str = ""
|
|
58
|
+
lookups: frozenset[str] = frozenset()
|
|
59
|
+
guessed: bool = False
|
|
54
60
|
|
|
55
61
|
@property
|
|
56
62
|
def singular(self) -> str:
|
|
57
63
|
return Names.snake(self.source)
|
|
58
64
|
|
|
59
|
-
def field_for(self, argument: str) -> str | None:
|
|
60
|
-
"""The
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if
|
|
64
|
-
return self.key
|
|
65
|
-
if snake.endswith("_id") and self.singular.endswith(snake[:-3]):
|
|
66
|
-
return self.key
|
|
65
|
+
def field_for(self, argument: str, own: bool) -> str | None:
|
|
66
|
+
"""The row field a path argument or query parameter selects: the key when the
|
|
67
|
+
argument is the path's own segment and names it, else a scalar field of that
|
|
68
|
+
name (spelled the vendor's way, ``doi`` → ``DOI``)."""
|
|
69
|
+
if own and self._names_key(argument):
|
|
70
|
+
return self.key_path or self.key
|
|
67
71
|
legal = self.fields.get(argument, Names.field(argument))
|
|
68
72
|
if legal in self.filterable:
|
|
69
73
|
return legal
|
|
74
|
+
snake = Names.snake(argument)
|
|
75
|
+
spelled = next((f for f in sorted(self.filterable) if Names.snake(f) == snake), None)
|
|
76
|
+
if spelled is not None:
|
|
77
|
+
return spelled
|
|
70
78
|
prefixed = snake[len(self.singular) + 1 :] if snake.startswith(f"{self.singular}_") else ""
|
|
71
79
|
return prefixed if prefixed in self.filterable else None
|
|
72
80
|
|
|
81
|
+
def _names_key(self, argument: str) -> bool:
|
|
82
|
+
snake = Names.snake(argument)
|
|
83
|
+
if argument in self.lookups or snake in {"id", self.singular, f"{self.singular}_id"}:
|
|
84
|
+
return True
|
|
85
|
+
return snake.endswith("_id") and self.singular.endswith(snake[:-3])
|
|
86
|
+
|
|
73
87
|
def summary(self) -> dict:
|
|
74
88
|
return {"name": self.name, "key": self.key, "source": self.source, "pointer": self.pointer}
|
|
75
89
|
|
|
76
90
|
|
|
77
91
|
@dataclass(frozen=True)
|
|
78
92
|
class Shape:
|
|
79
|
-
"""What a response or request body holds:
|
|
93
|
+
"""What a response or request body holds: rows of an entity, many or one, at a path."""
|
|
80
94
|
|
|
81
95
|
entity: Entity
|
|
82
96
|
many: bool
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
class Shapes:
|
|
87
|
-
"""Response and body schemas of the document read as entity shapes."""
|
|
88
|
-
|
|
89
|
-
def __init__(self, document: OpenApiDocument, entities: dict[str, Entity]):
|
|
90
|
-
self.document = document
|
|
91
|
-
self.entities = entities
|
|
92
|
-
|
|
93
|
-
def _named(self, schema: Any) -> Entity | None:
|
|
94
|
-
seen: set[str] = set()
|
|
95
|
-
while isinstance(schema, dict) and isinstance(schema.get("$ref"), str) and schema["$ref"] not in seen:
|
|
96
|
-
seen.add(schema["$ref"])
|
|
97
|
-
name = self.document.schema_name(schema["$ref"])
|
|
98
|
-
if name in self.entities:
|
|
99
|
-
return self.entities[name]
|
|
100
|
-
schema = self.document.lookup(schema["$ref"], "/paths")
|
|
101
|
-
return None
|
|
102
|
-
|
|
103
|
-
def of(self, schema: Any, at: str) -> Shape | None:
|
|
104
|
-
if not isinstance(schema, dict):
|
|
105
|
-
return None
|
|
106
|
-
entity = self._named(schema)
|
|
107
|
-
if entity is not None:
|
|
108
|
-
return Shape(entity, False)
|
|
109
|
-
node = self.document.resolve(schema, at)
|
|
110
|
-
if not isinstance(node, dict):
|
|
111
|
-
return None
|
|
112
|
-
if node.get("type") == "array":
|
|
113
|
-
items = self._named(node.get("items"))
|
|
114
|
-
return Shape(items, True) if items is not None else None
|
|
115
|
-
properties = node.get("properties")
|
|
116
|
-
if not isinstance(properties, dict):
|
|
117
|
-
return None
|
|
118
|
-
wrapped = [(key, self._array_of(value)) for key, value in properties.items()]
|
|
119
|
-
found = [(key, entity) for key, entity in wrapped if entity is not None]
|
|
120
|
-
if len(found) != 1:
|
|
121
|
-
return None
|
|
122
|
-
key, entity = found[0]
|
|
123
|
-
return Shape(entity, True, key)
|
|
97
|
+
payload: Payload
|
|
124
98
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
return None
|
|
129
|
-
return self._named(node.get("items"))
|
|
99
|
+
@property
|
|
100
|
+
def envelope(self) -> str:
|
|
101
|
+
return self.payload.envelope
|
|
130
102
|
|
|
131
103
|
|
|
132
104
|
@dataclass(frozen=True)
|
|
@@ -176,6 +148,10 @@ class OperationPlan:
|
|
|
176
148
|
envelope: str | None = None
|
|
177
149
|
generate: dict[str, str] = field(default_factory=dict)
|
|
178
150
|
reason: str | None = None
|
|
151
|
+
payload: Payload | None = None
|
|
152
|
+
paging: RoutePaging | None = None
|
|
153
|
+
wrapper: dict = field(default_factory=dict)
|
|
154
|
+
page: str | None = None
|
|
179
155
|
|
|
180
156
|
def summary(self) -> dict:
|
|
181
157
|
out: dict[str, Any] = {
|
|
@@ -226,6 +202,10 @@ class OperationPlan:
|
|
|
226
202
|
lines.append(f"filter = {_inline(self.filter)}")
|
|
227
203
|
if self.generate:
|
|
228
204
|
lines.append(f"generate = {_inline(self.generate)}")
|
|
205
|
+
if self.wrapper:
|
|
206
|
+
lines.append(f"wrapper = {_s(json.dumps(self.wrapper, ensure_ascii=False))}")
|
|
207
|
+
if self.page:
|
|
208
|
+
lines.append(f"page = {_s(self.page)}")
|
|
229
209
|
return lines
|
|
230
210
|
|
|
231
211
|
|
|
@@ -241,9 +221,16 @@ def _inline(mapping: dict[str, str]) -> str:
|
|
|
241
221
|
class RoutePlanner:
|
|
242
222
|
"""Every selected operation of the document, planned in document order."""
|
|
243
223
|
|
|
244
|
-
def __init__(
|
|
224
|
+
def __init__(
|
|
225
|
+
self,
|
|
226
|
+
document: OpenApiDocument,
|
|
227
|
+
entities: dict[str, Entity],
|
|
228
|
+
selected: list[OpenApiOperation],
|
|
229
|
+
payloads: Payloads,
|
|
230
|
+
):
|
|
245
231
|
self.document = document
|
|
246
|
-
self.
|
|
232
|
+
self.entities = entities
|
|
233
|
+
self.payloads = payloads
|
|
247
234
|
self.selected = selected
|
|
248
235
|
self.names = Namespace(limit=MAX_OPERATION_NAME)
|
|
249
236
|
self.warnings: list[str] = []
|
|
@@ -265,8 +252,6 @@ class RoutePlanner:
|
|
|
265
252
|
)
|
|
266
253
|
return self
|
|
267
254
|
|
|
268
|
-
# ------------------------------------------------------------- one operation
|
|
269
|
-
|
|
270
255
|
def _plan_one(self, operation: OpenApiOperation) -> None:
|
|
271
256
|
unservable = self._unservable_path(operation.path)
|
|
272
257
|
if unservable is not None:
|
|
@@ -327,6 +312,8 @@ class RoutePlanner:
|
|
|
327
312
|
plan.action = "read"
|
|
328
313
|
plan.status = None
|
|
329
314
|
plan.envelope = None
|
|
315
|
+
plan.payload = None
|
|
316
|
+
plan.paging = None
|
|
330
317
|
unbound = [p for p in placeholders if p not in plan.filter and p not in plan.params]
|
|
331
318
|
plan.params = (*plan.params, *unbound)
|
|
332
319
|
return plan
|
|
@@ -350,16 +337,15 @@ class RoutePlanner:
|
|
|
350
337
|
params.append(parameter.name)
|
|
351
338
|
return tuple(dict.fromkeys(params))
|
|
352
339
|
|
|
340
|
+
def _shape(self, payload: Payload | None) -> Shape | None:
|
|
341
|
+
entity = self.entities.get(payload.item) if payload is not None else None
|
|
342
|
+
return Shape(entity, payload.many, payload) if entity is not None and payload is not None else None
|
|
343
|
+
|
|
353
344
|
def _response_shape(self, operation: OpenApiOperation) -> Shape | None:
|
|
354
|
-
|
|
355
|
-
if success is None:
|
|
356
|
-
return None
|
|
357
|
-
return self.shapes.of(success[1], f"{operation.pointer}/responses/{success[0]}")
|
|
345
|
+
return self._shape(self.payloads.responses.get(operation.pointer))
|
|
358
346
|
|
|
359
347
|
def _body_shape(self, operation: OpenApiOperation) -> Shape | None:
|
|
360
|
-
return self.
|
|
361
|
-
|
|
362
|
-
# ------------------------------------------------------------- per method
|
|
348
|
+
return self._shape(self.payloads.bodies.get(operation.pointer))
|
|
363
349
|
|
|
364
350
|
def _read(self, operation: OpenApiOperation, plan: OperationPlan, placeholders: tuple[str, ...]) -> str | None:
|
|
365
351
|
shape = self._response_shape(operation)
|
|
@@ -368,7 +354,16 @@ class RoutePlanner:
|
|
|
368
354
|
plan.entity = shape.entity.name
|
|
369
355
|
plan.single = not shape.many
|
|
370
356
|
plan.envelope = shape.envelope or None
|
|
371
|
-
plan.
|
|
357
|
+
plan.payload = shape.payload
|
|
358
|
+
plan.filter = self._filters(shape.entity, operation.path, placeholders, plan.params if shape.many else ())
|
|
359
|
+
if shape.many:
|
|
360
|
+
rows_at = shape.payload.wrappers[-1] if shape.payload.wrappers else None
|
|
361
|
+
plan.paging = RoutePaging.of(operation, rows_at)
|
|
362
|
+
if plan.single and placeholders and not plan.filter:
|
|
363
|
+
return (
|
|
364
|
+
f"path argument {{{placeholders[-1]}}} names no field of {shape.entity.name}, "
|
|
365
|
+
"so the row it asks for cannot be found"
|
|
366
|
+
)
|
|
372
367
|
return None
|
|
373
368
|
|
|
374
369
|
def _create(self, operation: OpenApiOperation, plan: OperationPlan, placeholders: tuple[str, ...]) -> str | None:
|
|
@@ -376,11 +371,12 @@ class RoutePlanner:
|
|
|
376
371
|
if shape is not None and shape.many:
|
|
377
372
|
plan.entity = shape.entity.name
|
|
378
373
|
plan.envelope = shape.envelope or None
|
|
379
|
-
plan.
|
|
374
|
+
plan.payload = shape.payload
|
|
375
|
+
plan.filter = self._filters(shape.entity, operation.path, placeholders, plan.params)
|
|
380
376
|
return None
|
|
381
377
|
read = self._reads.get(operation.path)
|
|
382
|
-
if read is not None and not read.many and self._filters(read.entity, placeholders, ()):
|
|
383
|
-
return self._write(plan, read.entity, placeholders, "update", self._status(operation, 200))
|
|
378
|
+
if read is not None and not read.many and self._filters(read.entity, operation.path, placeholders, ()):
|
|
379
|
+
return self._write(plan, read.entity, operation.path, placeholders, "update", self._status(operation, 200))
|
|
384
380
|
entity = self._entity_of(shape, self._body_shape(operation), read)
|
|
385
381
|
if entity is None:
|
|
386
382
|
return "neither the response nor the body is an entity schema"
|
|
@@ -388,7 +384,7 @@ class RoutePlanner:
|
|
|
388
384
|
plan.action = "create"
|
|
389
385
|
plan.single = True
|
|
390
386
|
plan.status = self._status(operation, 201)
|
|
391
|
-
plan.generate = {entity.key: "int" if entity.key_type == "integer" else "uuid"}
|
|
387
|
+
plan.generate = {entity.key: "int" if entity.key_type == "integer" else "uuid"} if not entity.key_path else {}
|
|
392
388
|
return None
|
|
393
389
|
|
|
394
390
|
def _update(self, operation: OpenApiOperation, plan: OperationPlan, placeholders: tuple[str, ...]) -> str | None:
|
|
@@ -397,18 +393,18 @@ class RoutePlanner:
|
|
|
397
393
|
)
|
|
398
394
|
if entity is None:
|
|
399
395
|
return "neither the response, the body nor the GET of this path names an entity"
|
|
400
|
-
return self._write(plan, entity, placeholders, "update", self._status(operation, 200))
|
|
396
|
+
return self._write(plan, entity, operation.path, placeholders, "update", self._status(operation, 200))
|
|
401
397
|
|
|
402
398
|
def _delete(self, operation: OpenApiOperation, plan: OperationPlan, placeholders: tuple[str, ...]) -> str | None:
|
|
403
399
|
entity = self._entity_of(self._reads.get(operation.path), self._response_shape(operation), None)
|
|
404
400
|
if entity is None:
|
|
405
401
|
return "neither the response nor the GET of this path names an entity"
|
|
406
|
-
return self._write(plan, entity, placeholders, "delete", self._status(operation, 204))
|
|
402
|
+
return self._write(plan, entity, operation.path, placeholders, "delete", self._status(operation, 204))
|
|
407
403
|
|
|
408
404
|
def _write(
|
|
409
|
-
self, plan: OperationPlan, entity: Entity, placeholders: tuple[str, ...], action: str, status: int
|
|
405
|
+
self, plan: OperationPlan, entity: Entity, path: str, placeholders: tuple[str, ...], action: str, status: int
|
|
410
406
|
) -> str | None:
|
|
411
|
-
filters = self._filters(entity, placeholders, ())
|
|
407
|
+
filters = self._filters(entity, path, placeholders, ())
|
|
412
408
|
if not filters:
|
|
413
409
|
return f"no path argument names a field of {entity.name}, so the row to {action} cannot be found"
|
|
414
410
|
plan.entity = entity.name
|
|
@@ -418,8 +414,6 @@ class RoutePlanner:
|
|
|
418
414
|
plan.filter = filters
|
|
419
415
|
return None
|
|
420
416
|
|
|
421
|
-
# ------------------------------------------------------------- helpers
|
|
422
|
-
|
|
423
417
|
@staticmethod
|
|
424
418
|
def _entity_of(*shapes: Shape | None) -> Entity | None:
|
|
425
419
|
for shape in shapes:
|
|
@@ -435,7 +429,8 @@ class RoutePlanner:
|
|
|
435
429
|
return int(success[0])
|
|
436
430
|
|
|
437
431
|
@staticmethod
|
|
438
|
-
def _filters(entity: Entity, placeholders: tuple[str, ...], params: tuple[str, ...]) -> dict[str, str]:
|
|
439
|
-
|
|
440
|
-
|
|
432
|
+
def _filters(entity: Entity, path: str, placeholders: tuple[str, ...], params: tuple[str, ...]) -> dict[str, str]:
|
|
433
|
+
own = own_segment(path)
|
|
434
|
+
arguments = [(p, p == own) for p in placeholders] + [(p, False) for p in params if p not in PAGING_PARAMS]
|
|
435
|
+
found = {argument: entity.field_for(argument, is_own) for argument, is_own in arguments}
|
|
441
436
|
return {argument: target for argument, target in found.items() if target is not None}
|