@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Text cells typed by the entity's schema — the one rule both CLIs apply to a CSV/TSV cell.
|
|
2
|
+
|
|
3
|
+
The TypeScript twin is ``worldData/cast.ts``; ``tests/conformance/csv_cast.json`` pins
|
|
4
|
+
them to the same answers. ``integer`` and ``number`` parse as numbers (a number is
|
|
5
|
+
always a float), ``boolean`` takes true/false/1/0 in any case, ``array`` and
|
|
6
|
+
``object`` parse as JSON of that shape, ``string`` is the text as given. An empty
|
|
7
|
+
cell is ``None`` when the field's type admits null, left out when the field is
|
|
8
|
+
optional, the empty string for a required string, and refused for any other
|
|
9
|
+
required field. A union type (``integer | string``) tries each member in order,
|
|
10
|
+
``string`` last. A cell that fits none is refused, naming the field and the cell;
|
|
11
|
+
a column the schema does not name stays text (the gate refuses it, as it always did).
|
|
12
|
+
|
|
13
|
+
Standard library only, so the bridge and the api bundle can load it.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import math
|
|
20
|
+
import re
|
|
21
|
+
from collections.abc import Callable, Iterable
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from .validation import fail
|
|
27
|
+
|
|
28
|
+
INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
|
|
29
|
+
NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
|
|
30
|
+
BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
|
|
31
|
+
MEMBERS = ("integer", "number", "boolean", "array", "object", "string", "null")
|
|
32
|
+
|
|
33
|
+
_MISSING = object()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class FieldType:
|
|
38
|
+
"""One field as ``gateway worlds describe`` reports it: ``type`` is the JSON Schema
|
|
39
|
+
type word(s) joined by ``" | "``, a ``$ref`` name, or None when the schema says nothing."""
|
|
40
|
+
|
|
41
|
+
name: str
|
|
42
|
+
type: str | None
|
|
43
|
+
required: bool
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _article(kind: str) -> str:
|
|
47
|
+
return "an" if kind[:1].lower() in "aeiou" else "a"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def refuse(where: str, field: str, cell: str | None, kind: str) -> None:
|
|
51
|
+
"""The row-level refusal a cell that will not cast earns (same words as the TypeScript twin)."""
|
|
52
|
+
if cell is None:
|
|
53
|
+
fail("rows", f"{where}: {field} is empty; the field is a required {kind}", where)
|
|
54
|
+
fail(
|
|
55
|
+
"rows",
|
|
56
|
+
f"{where}: {field} = {json.dumps(cell)} is not {_article(kind)} {kind}",
|
|
57
|
+
where,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def type_members(kind: str | None) -> list[str]:
|
|
62
|
+
"""The typed members of a field's ``type`` (``"integer | null"`` -> integer, null); an
|
|
63
|
+
unknown word (a ``$ref`` name) is dropped."""
|
|
64
|
+
if kind is None:
|
|
65
|
+
return []
|
|
66
|
+
parts = (part.strip().lower() for part in kind.split("|"))
|
|
67
|
+
return [part for part in parts if part in MEMBERS]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _integer(text: str) -> Any:
|
|
71
|
+
return int(text) if INTEGER_TEXT.match(text) else _MISSING
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _number(text: str) -> Any:
|
|
75
|
+
if not NUMBER_TEXT.match(text):
|
|
76
|
+
return _MISSING
|
|
77
|
+
value = float(text)
|
|
78
|
+
return value if math.isfinite(value) else _MISSING
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _boolean(text: str) -> Any:
|
|
82
|
+
return BOOLEANS.get(text.lower(), _MISSING)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _json_of(shape: type) -> Callable[[str], Any]:
|
|
86
|
+
def cast(text: str) -> Any:
|
|
87
|
+
try:
|
|
88
|
+
value = json.loads(text)
|
|
89
|
+
except ValueError:
|
|
90
|
+
return _MISSING
|
|
91
|
+
return value if isinstance(value, shape) else _MISSING
|
|
92
|
+
|
|
93
|
+
return cast
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _no_cast(_text: str) -> Any:
|
|
97
|
+
return _MISSING
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
_CAST: dict[str, Callable[[str], Any]] = {
|
|
101
|
+
"integer": _integer,
|
|
102
|
+
"number": _number,
|
|
103
|
+
"boolean": _boolean,
|
|
104
|
+
"array": _json_of(list),
|
|
105
|
+
"object": _json_of(dict),
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class FieldCaster:
|
|
110
|
+
"""How one field's cells cast."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, name: str, kind: str | None, required: bool):
|
|
113
|
+
self.name = name
|
|
114
|
+
self.type = kind
|
|
115
|
+
self.required = required
|
|
116
|
+
self.members = type_members(kind)
|
|
117
|
+
|
|
118
|
+
def cast(self, cell: str | None, where: str) -> Any:
|
|
119
|
+
"""The typed value; :data:`OMIT` to leave the field out of the row."""
|
|
120
|
+
if cell is None:
|
|
121
|
+
return self._empty(where)
|
|
122
|
+
if not self.members:
|
|
123
|
+
return cell
|
|
124
|
+
text = cell.strip()
|
|
125
|
+
for member in self.members:
|
|
126
|
+
value = _CAST.get(member, _no_cast)(text)
|
|
127
|
+
if value is not _MISSING:
|
|
128
|
+
return value
|
|
129
|
+
if "string" in self.members:
|
|
130
|
+
return cell
|
|
131
|
+
refuse(where, self.name, cell, self.type or "value")
|
|
132
|
+
|
|
133
|
+
def _empty(self, where: str) -> Any:
|
|
134
|
+
if not self.members or "null" in self.members:
|
|
135
|
+
return None
|
|
136
|
+
if not self.required:
|
|
137
|
+
return OMIT
|
|
138
|
+
if "string" in self.members:
|
|
139
|
+
return ""
|
|
140
|
+
refuse(where, self.name, None, self.type or "value")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
#: What :meth:`FieldCaster.cast` answers for an empty optional cell: the field stays out of the row.
|
|
144
|
+
OMIT = object()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class RowCaster:
|
|
148
|
+
"""The casters of one entity's fields; a column the entity does not name stays text."""
|
|
149
|
+
|
|
150
|
+
def __init__(self, fields: Iterable[FieldType]):
|
|
151
|
+
self._fields = {f.name: FieldCaster(f.name, f.type, f.required) for f in fields}
|
|
152
|
+
|
|
153
|
+
def row(self, record: dict[str, str | None], where: str) -> dict:
|
|
154
|
+
row: dict = {}
|
|
155
|
+
for name, cell in record.items():
|
|
156
|
+
caster = self._fields.get(name)
|
|
157
|
+
value = cell if caster is None else caster.cast(cell, where)
|
|
158
|
+
if value is not OMIT:
|
|
159
|
+
row[name] = value
|
|
160
|
+
return row
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
#: Text cells as the untyped reader always made them: the text, or None for an empty cell.
|
|
164
|
+
TEXT_CASTER = RowCaster([])
|
|
165
|
+
|
|
166
|
+
EntityCasters = Callable[[str], "RowCaster | None"]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def casters_of(entities: dict[str, list[FieldType]]) -> EntityCasters:
|
|
170
|
+
"""Casters by entity name."""
|
|
171
|
+
by_name = {name: RowCaster(fields) for name, fields in entities.items()}
|
|
172
|
+
return by_name.get
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _definition(world: dict, entity: Any) -> dict:
|
|
176
|
+
schema = entity.get("schema") if isinstance(entity, dict) else None
|
|
177
|
+
ref = schema.get("$ref") if isinstance(schema, dict) else None
|
|
178
|
+
if isinstance(ref, str) and ref.startswith("#/$defs/"):
|
|
179
|
+
definition = (world.get("$defs") or {}).get(ref[len("#/$defs/") :])
|
|
180
|
+
return definition if isinstance(definition, dict) else {}
|
|
181
|
+
# An inline payload (`schema entity add --schema '{…}'`) declares its fields right here.
|
|
182
|
+
if isinstance(schema, dict) and isinstance(schema.get("properties"), dict):
|
|
183
|
+
return schema
|
|
184
|
+
return {}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _field_type(field_schema: Any) -> str | None:
|
|
188
|
+
if not isinstance(field_schema, dict):
|
|
189
|
+
return None
|
|
190
|
+
kind = field_schema.get("type")
|
|
191
|
+
if isinstance(kind, list):
|
|
192
|
+
# A nullable field is `["string", "null"]` in JSON Schema; one readable type, not the list.
|
|
193
|
+
return " | ".join(str(part) for part in kind)
|
|
194
|
+
if kind is None and isinstance(field_schema.get("$ref"), str):
|
|
195
|
+
return field_schema["$ref"].rsplit("/", 1)[-1]
|
|
196
|
+
return kind if isinstance(kind, str) else None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def world_fields(world: dict) -> dict[str, list[FieldType]]:
|
|
200
|
+
"""Every entity's fields as a world.json document declares them (the ``$defs`` entry its
|
|
201
|
+
``schema.$ref`` names, or an inline ``properties``), sorted by entity name — the field
|
|
202
|
+
list ``gateway worlds describe`` reports and a CSV import casts by."""
|
|
203
|
+
gateway = world.get("x-gateway") if isinstance(world, dict) else None
|
|
204
|
+
entities = (gateway.get("entities") or {}) if isinstance(gateway, dict) else {}
|
|
205
|
+
fields: dict[str, list[FieldType]] = {}
|
|
206
|
+
for name, entity in sorted(entities.items()):
|
|
207
|
+
definition = _definition(world, entity)
|
|
208
|
+
required = set(definition.get("required") or [])
|
|
209
|
+
properties = definition.get("properties") or {}
|
|
210
|
+
fields[name] = [
|
|
211
|
+
FieldType(field_name, _field_type(field_schema), field_name in required)
|
|
212
|
+
for field_name, field_schema in properties.items()
|
|
213
|
+
]
|
|
214
|
+
return fields
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def world_casters(world: dict) -> EntityCasters:
|
|
218
|
+
""":func:`casters_of` for a world.json document."""
|
|
219
|
+
return casters_of(world_fields(world))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def schema_fields(schema: Any) -> list[FieldType]:
|
|
223
|
+
"""The fields of one resolved entity schema (``properties`` + ``required``)."""
|
|
224
|
+
if not isinstance(schema, dict):
|
|
225
|
+
return []
|
|
226
|
+
required = set(schema.get("required") or [])
|
|
227
|
+
return [
|
|
228
|
+
FieldType(name, _field_type(field_schema), name in required)
|
|
229
|
+
for name, field_schema in (schema.get("properties") or {}).items()
|
|
230
|
+
]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def contract_casters(entities: dict) -> EntityCasters:
|
|
234
|
+
""":func:`casters_of` for a compiled contract's ``entities`` (each carries its resolved
|
|
235
|
+
``schema``) — the local twin of :func:`world_casters`."""
|
|
236
|
+
return casters_of({name: schema_fields(entity.get("schema")) for name, entity in entities.items()})
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def tree_casters(root: str | Path) -> EntityCasters:
|
|
240
|
+
""":func:`contract_casters` for a working tree: what a ``.csv``/``.tsv`` imported into the
|
|
241
|
+
directory is typed by. A tree that does not compile raises the compiler's own ``ContractError``."""
|
|
242
|
+
from .compiler import compile_bundle
|
|
243
|
+
|
|
244
|
+
return contract_casters(compile_bundle(root, snapshots=False).entities)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def described_casters(described: dict) -> EntityCasters:
|
|
248
|
+
""":func:`casters_of` for what ``GET /api/public/worlds/{slug}`` (``gateway worlds describe``)
|
|
249
|
+
answers: ``entities: [{name, fields: [{name, type, required}]}]`` — the hosted twin of
|
|
250
|
+
:func:`contract_casters`, the same fields the TypeScript CLI casts by."""
|
|
251
|
+
entities: dict[str, list[FieldType]] = {}
|
|
252
|
+
for entity in described.get("entities") or []:
|
|
253
|
+
if not isinstance(entity, dict) or not isinstance(entity.get("name"), str):
|
|
254
|
+
continue
|
|
255
|
+
entities[entity["name"]] = [
|
|
256
|
+
FieldType(
|
|
257
|
+
str(field.get("name")),
|
|
258
|
+
field.get("type") if isinstance(field.get("type"), str) else None,
|
|
259
|
+
bool(field.get("required")),
|
|
260
|
+
)
|
|
261
|
+
for field in entity.get("fields") or []
|
|
262
|
+
if isinstance(field, dict) and isinstance(field.get("name"), str)
|
|
263
|
+
]
|
|
264
|
+
return casters_of(entities)
|
|
@@ -11,6 +11,7 @@ from typing import Any
|
|
|
11
11
|
from .._toml import tomllib
|
|
12
12
|
from .validation import (
|
|
13
13
|
LANGUAGE,
|
|
14
|
+
MAX_ENTITY_NAME,
|
|
14
15
|
ContractError,
|
|
15
16
|
Resolver,
|
|
16
17
|
canonical,
|
|
@@ -59,6 +60,13 @@ def identifier(value: Any, location: str) -> str:
|
|
|
59
60
|
"Use a SQL-safe name; _, sqlite_, gateway_, benchmark_ prefixes are reserved",
|
|
60
61
|
location,
|
|
61
62
|
)
|
|
63
|
+
if len(value) > MAX_ENTITY_NAME:
|
|
64
|
+
fail(
|
|
65
|
+
"identity",
|
|
66
|
+
f"entity name must be at most {MAX_ENTITY_NAME} characters, got {len(value)}",
|
|
67
|
+
location,
|
|
68
|
+
limit=MAX_ENTITY_NAME,
|
|
69
|
+
)
|
|
62
70
|
return value
|
|
63
71
|
|
|
64
72
|
|
|
@@ -244,6 +252,9 @@ class Contract:
|
|
|
244
252
|
metadata: dict
|
|
245
253
|
artifacts: dict[str, bytes]
|
|
246
254
|
lock: dict
|
|
255
|
+
#: Structured, non-fatal findings (``recursive_reference``: a $defs cycle no
|
|
256
|
+
#: entity reaches); never part of the lock or the artifacts.
|
|
257
|
+
warnings: tuple[dict, ...] = ()
|
|
247
258
|
|
|
248
259
|
@property
|
|
249
260
|
def target(self) -> dict:
|
|
@@ -263,7 +274,12 @@ def build(bundle: Path) -> Contract:
|
|
|
263
274
|
manifest = bundle_mode(bundle)
|
|
264
275
|
if manifest is None:
|
|
265
276
|
fail("language", "Not a schema-backed bundle; declare [schema] language")
|
|
277
|
+
from .cycles import ReferenceCycles
|
|
278
|
+
from .route_bindings import check_route_bindings
|
|
279
|
+
|
|
266
280
|
resolver = Resolver(bundle)
|
|
281
|
+
cycles = ReferenceCycles(resolver)
|
|
282
|
+
warnings: list[dict] = []
|
|
267
283
|
document = resolver.load("world.json")
|
|
268
284
|
annotation = document.get("x-gateway")
|
|
269
285
|
if not isinstance(annotation, dict):
|
|
@@ -312,6 +328,7 @@ def build(bundle: Path) -> Contract:
|
|
|
312
328
|
location,
|
|
313
329
|
)
|
|
314
330
|
payload = resolver.resolve(entity["schema"], "world.json", f"{location}/schema")
|
|
331
|
+
warnings.extend(cycles.reached(name, entity["schema"], f"{location}/schema"))
|
|
315
332
|
if (
|
|
316
333
|
payload.get("type") != "object"
|
|
317
334
|
or payload.get("additionalProperties") is not False
|
|
@@ -382,6 +399,7 @@ def build(bundle: Path) -> Contract:
|
|
|
382
399
|
"relationships": sorted(relationships, key=canonical),
|
|
383
400
|
"location": location,
|
|
384
401
|
}
|
|
402
|
+
warnings.extend(cycles.unreached(document.get("$defs", {}), "world.json"))
|
|
385
403
|
for name, entity in compiled.items():
|
|
386
404
|
for rel in entity["relationships"]:
|
|
387
405
|
location = f"{entity['location']}/relationships"
|
|
@@ -524,7 +542,9 @@ def build(bundle: Path) -> Contract:
|
|
|
524
542
|
"tools_hash": digest(encoded(compiled_tools)),
|
|
525
543
|
"artifacts": {p: digest(b) for p, b in sorted(artifacts.items())},
|
|
526
544
|
}
|
|
527
|
-
|
|
545
|
+
contract = Contract(metadata, artifacts, lock, tuple(warnings))
|
|
546
|
+
check_route_bindings(bundle, contract)
|
|
547
|
+
return contract
|
|
528
548
|
|
|
529
549
|
|
|
530
550
|
SCALARS = {"string", "integer", "number", "boolean"}
|
|
@@ -609,6 +629,7 @@ def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
|
|
|
609
629
|
(`gateway worlds schema compile` reports it at once, see ``compile_world``)."""
|
|
610
630
|
bundle = Path(bundle)
|
|
611
631
|
contract = build(bundle)
|
|
632
|
+
bind_connection(bundle, contract)
|
|
612
633
|
for path, content in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
|
|
613
634
|
destination = bundle / path
|
|
614
635
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -694,12 +715,25 @@ def compile_world(bundle: str | Path) -> tuple[Contract, dict]:
|
|
|
694
715
|
return contract, build_all(bundle, contract)
|
|
695
716
|
|
|
696
717
|
|
|
718
|
+
def bind_connection(bundle: Path, contract: Contract) -> None:
|
|
719
|
+
"""A world with a connector.toml: every route lands on an entity the contract
|
|
720
|
+
declares (:meth:`Spec.check_entities`). Runs before compile writes and before
|
|
721
|
+
check trusts the artifacts, so a ghost entity is refused, never served."""
|
|
722
|
+
spec_path = bundle / "connector.toml"
|
|
723
|
+
if not spec_path.is_file():
|
|
724
|
+
return
|
|
725
|
+
from .connector import parse_spec
|
|
726
|
+
|
|
727
|
+
parse_spec(spec_path.read_text(encoding="utf-8")).check_entities(contract.entities)
|
|
728
|
+
|
|
729
|
+
|
|
697
730
|
def check_bundle(bundle: str | Path) -> Contract:
|
|
698
731
|
"""The contract, provided every generated artifact and every snapshot present in
|
|
699
732
|
the tree matches it; a snapshot's sidecar must name this contract and the
|
|
700
733
|
file's own sha256. Absent snapshots are not required: the legacy path serves."""
|
|
701
734
|
bundle = Path(bundle)
|
|
702
735
|
contract = build(bundle)
|
|
736
|
+
bind_connection(bundle, contract)
|
|
703
737
|
for path, expected in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
|
|
704
738
|
try:
|
|
705
739
|
matches = (bundle / path).read_bytes() == expected
|
|
@@ -48,20 +48,38 @@ PIN = "conform"
|
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
def load_openapi(path: str | Path) -> dict:
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
51
|
+
return parse_openapi(Path(path).read_text(encoding="utf-8"), str(path))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def parse_openapi(text: str, name: str, *, location: str = "/conform/openapi") -> dict:
|
|
55
|
+
"""The one OpenAPI reader: YAML when the name says so or the text is not JSON,
|
|
56
|
+
JSON otherwise. ``conform`` and ``schema init --from-openapi`` both read through
|
|
57
|
+
here, so a document parses the same way wherever it came from."""
|
|
58
|
+
is_json = name.endswith(".json") or (not name.endswith((".yaml", ".yml")) and text.lstrip()[:1] in ("{", "["))
|
|
59
|
+
document = _parse_json(text, name, location) if is_json else _parse_yaml(text, name, location)
|
|
60
60
|
if not isinstance(document, dict) or "paths" not in document:
|
|
61
|
-
fail("conform", f"{
|
|
61
|
+
fail("conform", f"{name} is not an OpenAPI document (no paths)", location)
|
|
62
62
|
return document
|
|
63
63
|
|
|
64
64
|
|
|
65
|
+
def _parse_json(text: str, name: str, location: str) -> Any:
|
|
66
|
+
try:
|
|
67
|
+
return json.loads(text)
|
|
68
|
+
except ValueError as error:
|
|
69
|
+
fail("conform", f"{name} is not a JSON document: {error}", location)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _parse_yaml(text: str, name: str, location: str) -> Any:
|
|
73
|
+
try:
|
|
74
|
+
import yaml # optional at runtime; the SDK depends on it, the vendored server does not
|
|
75
|
+
except ImportError: # pragma: no cover
|
|
76
|
+
fail("conform", "Reading a YAML document needs PyYAML; convert it to JSON", location)
|
|
77
|
+
try:
|
|
78
|
+
return yaml.safe_load(text)
|
|
79
|
+
except yaml.YAMLError as error:
|
|
80
|
+
fail("conform", f"{name} is not a YAML document: {error}", location)
|
|
81
|
+
|
|
82
|
+
|
|
65
83
|
def server_prefixes(document: dict) -> list[str]:
|
|
66
84
|
"""The path part of each server URL, longest first, so a world whose paths carry
|
|
67
85
|
the vendor's prefix (``/investigations-v2/breach/...``) still lines up."""
|
|
@@ -72,13 +72,12 @@ import urllib.request
|
|
|
72
72
|
from dataclasses import dataclass, field
|
|
73
73
|
from datetime import datetime, timezone
|
|
74
74
|
from pathlib import Path
|
|
75
|
-
from typing import Any, Callable
|
|
75
|
+
from typing import Any, Callable, Iterable
|
|
76
76
|
|
|
77
|
-
from .._toml import tomllib
|
|
78
77
|
from .compiler import check_bundle, compile_bundle
|
|
79
78
|
from .store import SQLiteStore
|
|
80
79
|
from .transforms import apply_transforms
|
|
81
|
-
from .validation import ContractError, fail
|
|
80
|
+
from .validation import ContractError, fail, operation_name, parse_toml
|
|
82
81
|
|
|
83
82
|
SPEC = "connector.toml"
|
|
84
83
|
CAPTURES = "captures"
|
|
@@ -87,7 +86,6 @@ SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
|
|
|
87
86
|
DEFAULT_TOKEN_PATH = "/oauth/token"
|
|
88
87
|
METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
89
88
|
SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
90
|
-
NAME = re.compile(r"^[a-z][a-z0-9_]*$")
|
|
91
89
|
#: Vendor-named things — query parameters, path arguments, redacted fields — keep the
|
|
92
90
|
#: vendor's casing (``startDate``, ``Limit``); only our own operation names are snake_case.
|
|
93
91
|
FIELD = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
|
|
@@ -178,6 +176,29 @@ class Spec:
|
|
|
178
176
|
fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
|
|
179
177
|
return self.operations[name]
|
|
180
178
|
|
|
179
|
+
def check_entities(self, declared: Iterable[str]) -> None:
|
|
180
|
+
"""Every operation that lands on an entity lands on one the world declares.
|
|
181
|
+
|
|
182
|
+
``parse_spec`` reads connector.toml alone, so this is the step that binds the
|
|
183
|
+
connection to ``schema/world.json``; compile, check and the platform's describe
|
|
184
|
+
all call it, so a route onto a ghost entity is refused before it is served."""
|
|
185
|
+
known = set(declared)
|
|
186
|
+
for index, operation in enumerate(self.operations.values()):
|
|
187
|
+
landings = [(operation.entity, f"/operations/{index}/entity")] + [
|
|
188
|
+
(child.entity, f"/operations/{index}/ingest/explode/{position}/entity")
|
|
189
|
+
for position, child in enumerate(operation.explode)
|
|
190
|
+
]
|
|
191
|
+
for entity, location in landings:
|
|
192
|
+
if entity is None or entity in known:
|
|
193
|
+
continue
|
|
194
|
+
fail(
|
|
195
|
+
"spec",
|
|
196
|
+
f"operation {operation.name} lands on entity {entity}, which schema/world.json "
|
|
197
|
+
f"does not declare (declared: {', '.join(sorted(known)) or 'none'}); add the entity with "
|
|
198
|
+
"`gateway worlds schema entity add`, or point the route at one that exists",
|
|
199
|
+
location,
|
|
200
|
+
)
|
|
201
|
+
|
|
181
202
|
|
|
182
203
|
# ---------------------------------------------------------------- spec
|
|
183
204
|
|
|
@@ -300,10 +321,11 @@ def _positive(table: dict, key: str, location: str, default: int | float, *, zer
|
|
|
300
321
|
|
|
301
322
|
|
|
302
323
|
def parse_spec(text: str) -> Spec:
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
324
|
+
return spec_from(parse_toml(text))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def spec_from(raw: dict) -> Spec:
|
|
328
|
+
"""The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
|
|
307
329
|
connector = _table(raw.get("connector"), "/connector")
|
|
308
330
|
slug = _string(connector, "slug", "/connector")
|
|
309
331
|
if not SLUG.match(slug):
|
|
@@ -356,8 +378,7 @@ def parse_spec(text: str) -> Spec:
|
|
|
356
378
|
location = f"/operations/{index}"
|
|
357
379
|
table = _table(entry, location)
|
|
358
380
|
name = _string(table, "name", location)
|
|
359
|
-
|
|
360
|
-
fail("spec", "operation name must be snake_case", f"{location}/name")
|
|
381
|
+
operation_name(name, f"{location}/name")
|
|
361
382
|
if name in operations:
|
|
362
383
|
fail("spec", f"operation {name} is declared twice", f"{location}/name")
|
|
363
384
|
methods = _verbs(table, location)
|
|
@@ -369,14 +390,20 @@ def parse_spec(text: str) -> Spec:
|
|
|
369
390
|
path = _string(table, "path", location)
|
|
370
391
|
if not path.startswith("/") or "?" in path or "#" in path:
|
|
371
392
|
fail("spec", "path must start with / and carry no query", f"{location}/path")
|
|
393
|
+
placeholders = PLACEHOLDER.findall(path)
|
|
394
|
+
repeated = [p for p in placeholders if placeholders.count(p) > 1]
|
|
395
|
+
if repeated:
|
|
396
|
+
fail(
|
|
397
|
+
"spec",
|
|
398
|
+
f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
|
|
399
|
+
f"{location}/path",
|
|
400
|
+
argument=repeated[0],
|
|
401
|
+
)
|
|
372
402
|
params = _names(table, "params", location, PARAM)
|
|
373
403
|
entity = _string(table, "entity", location, required=False)
|
|
374
404
|
results = table.get("results", "")
|
|
375
405
|
if not isinstance(results, str):
|
|
376
406
|
fail("spec", "results must be a dotted path string", f"{location}/results")
|
|
377
|
-
for placeholder in PLACEHOLDER.findall(path):
|
|
378
|
-
if placeholder in params:
|
|
379
|
-
fail("spec", f"{placeholder} is both a path argument and a query parameter", location)
|
|
380
407
|
drop, explode, carry_args = _ingest_projection(table.get("ingest", {}), f"{location}/ingest")
|
|
381
408
|
if (drop or explode or carry_args) and not entity:
|
|
382
409
|
fail("spec", "ingest projection needs an entity to land rows in", f"{location}/ingest")
|
|
@@ -457,11 +484,29 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
|
|
|
457
484
|
return drop, tuple(explode), carry_args
|
|
458
485
|
|
|
459
486
|
|
|
487
|
+
def bind_spec(spec: Spec, world_document: object) -> Spec:
|
|
488
|
+
"""``spec`` checked against a parsed world.json (:meth:`Spec.check_entities`); a
|
|
489
|
+
document without a usable ``x-gateway.entities`` map declares nothing."""
|
|
490
|
+
annotation = world_document.get("x-gateway") if isinstance(world_document, dict) else None
|
|
491
|
+
entities = annotation.get("entities") if isinstance(annotation, dict) else None
|
|
492
|
+
spec.check_entities(entities if isinstance(entities, dict) else {})
|
|
493
|
+
return spec
|
|
494
|
+
|
|
495
|
+
|
|
460
496
|
def load_spec(world: str | Path) -> Spec:
|
|
497
|
+
"""The world's connection, bound to its schema when the tree has one: a route
|
|
498
|
+
onto an entity ``schema/world.json`` does not declare is refused here too, so
|
|
499
|
+
serve, open, `worlds test`, describe and ingest read the same rule compile and check do."""
|
|
461
500
|
path = Path(world) / SPEC
|
|
462
501
|
if not path.is_file():
|
|
463
502
|
fail("spec", f"No {SPEC} in the world; describe the connection first", f"/{SPEC}")
|
|
464
|
-
|
|
503
|
+
spec = parse_spec(path.read_text(encoding="utf-8"))
|
|
504
|
+
world_json = Path(world) / "schema/world.json"
|
|
505
|
+
if not world_json.is_file():
|
|
506
|
+
return spec
|
|
507
|
+
from .validation import read_json
|
|
508
|
+
|
|
509
|
+
return bind_spec(spec, read_json(world_json))
|
|
465
510
|
|
|
466
511
|
|
|
467
512
|
# ---------------------------------------------------------------- requests
|
|
@@ -1,4 +1,41 @@
|
|
|
1
|
-
"""Custom connector world tools;
|
|
1
|
+
"""Custom connector world tools; keep writes on the store.
|
|
2
|
+
|
|
3
|
+
Declare each tool under x-gateway.tools in schema/world.json and in
|
|
4
|
+
[tools].serve of gateway-env.toml, then implement it here:
|
|
5
|
+
|
|
6
|
+
from gateway_env import store, tool
|
|
7
|
+
|
|
8
|
+
@tool(name="list_records")
|
|
9
|
+
def list_records(kind):
|
|
10
|
+
return [r for r in store.list("records") if r["kind"] == kind]
|
|
11
|
+
|
|
12
|
+
`store` (from gateway_env) is the world's rows. It offers exactly these methods;
|
|
13
|
+
`gateway worlds schema check .` refuses any other `store.<name>` at file:line, and a
|
|
14
|
+
hosted call into such a handler answers 400 {error: "handler", detail, handler}:
|
|
15
|
+
|
|
16
|
+
get(entity: str, key: dict) -> dict | None
|
|
17
|
+
One row of `entity` by its primary key, or None when no row has that key.
|
|
18
|
+
list(entity: str) -> list[dict]
|
|
19
|
+
Every row of `entity`, ordered by key.
|
|
20
|
+
list_where(entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]
|
|
21
|
+
A page of `entity` rows whose every `where` field holds one of the listed values, plus the total match count.
|
|
22
|
+
insert(entity: str, value: dict, *, source: dict | None = None) -> dict
|
|
23
|
+
Validate `value` against the entity's schema and add it; the key must be new.
|
|
24
|
+
update(entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict
|
|
25
|
+
Merge `patch` into the row at `key` and validate the result; returns the row.
|
|
26
|
+
delete(entity: str, key: dict, *, source: dict | None = None) -> None
|
|
27
|
+
Remove the row at `key`; a missing row is a contract error.
|
|
28
|
+
apply_batch(batch: dict) -> dict
|
|
29
|
+
Apply one ingest batch {id, target, mapping, coverage, mutations} atomically; an exact retry is a no-op.
|
|
30
|
+
lineage(entity: str, key: dict) -> list[dict]
|
|
31
|
+
The batches that wrote the row at `key`, oldest first, with each batch's source.
|
|
32
|
+
counts() -> dict[str, int]
|
|
33
|
+
Rows per entity, as {entity: count}.
|
|
34
|
+
|
|
35
|
+
A handler that raises AttributeError, TypeError or NameError is answered 400 with
|
|
36
|
+
that structured body; any other exception is answered 500 {error: "engine", ...}.
|
|
37
|
+
`schema init --connector custom --with-example-tool` writes the example in full.
|
|
38
|
+
"""
|
|
2
39
|
|
|
3
40
|
from gateway_env import store, tool
|
|
4
41
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# How the platform fills a world on this contract from a live Merge connection
|
|
2
|
-
# (`gateway worlds create --
|
|
2
|
+
# (`gateway worlds create <slug> --from-connection <id|google_drive>`, refreshed with
|
|
3
|
+
# `gateway worlds data pull <slug> --connection <id|google_drive>`; MCP `create_world` /
|
|
4
|
+
# `pull_world_data`).
|
|
3
5
|
# Each [[pulls]] is one read tool of Merge's `google_drive` tool pack, called on
|
|
4
6
|
# the connection's registered user; readers/merge.ingest.toml maps what comes back
|
|
5
7
|
# onto the world's entities. Only read tools belong here.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""How a ``basic`` world's credential rides a request: as the Basic *user*,
|
|
2
|
+
with an empty password (``Basic b64("<key>:")``). The host and the local
|
|
3
|
+
server present it that way (``api.vendor_credential``), and the world
|
|
4
|
+
(``WorldApp.authorized``) and the host's gate (``vendor_shaped_token``) read
|
|
5
|
+
it back with :func:`basic_user`, so the two sides cannot drift."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def basic_user(authorization: str) -> str | None:
|
|
13
|
+
"""The user of a ``Basic`` authorization header, or None when it is not one."""
|
|
14
|
+
if not authorization.lower().startswith("basic "):
|
|
15
|
+
return None
|
|
16
|
+
try:
|
|
17
|
+
decoded = base64.b64decode(authorization[6:].strip().encode("ascii"), validate=True).decode("utf-8")
|
|
18
|
+
except (ValueError, UnicodeDecodeError):
|
|
19
|
+
return None
|
|
20
|
+
return decoded.split(":", 1)[0]
|