@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Where an OpenAPI document comes from, for ``schema init --from-openapi``.
|
|
2
|
+
|
|
3
|
+
A source is a local file, a public ``http(s)://`` URL or text handed over by a
|
|
4
|
+
caller that already fetched it (the TS CLI reading ``drive:<path>`` through the
|
|
5
|
+
platform). Whatever the kind, the outcome is one :class:`OpenApiSource`: the
|
|
6
|
+
bytes' text, their sha256, and a label safe to record in ``provenance.json``
|
|
7
|
+
(a URL stripped of its userinfo and query string, so a token in a signed URL
|
|
8
|
+
never lands in the tree). Parsing goes through :func:`conform.parse_openapi`,
|
|
9
|
+
the package's one OpenAPI reader.
|
|
10
|
+
|
|
11
|
+
The URL fetch runs where the command runs (the caller's machine for
|
|
12
|
+
``gateway``, the sandbox for the Python ``gateway`` there) and is guarded the
|
|
13
|
+
way the web pod's ``fetchGuarded`` is: http(s) only, the host and every address
|
|
14
|
+
it resolves to public, redirects followed by hand with the same check per hop,
|
|
15
|
+
the body capped while it streams. Twin: ``gateway/web/src/features/agent/server/tools/common/webFetch.ts``.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
import ipaddress
|
|
22
|
+
import re
|
|
23
|
+
import socket
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.parse
|
|
26
|
+
import urllib.request
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from .conform import parse_openapi
|
|
31
|
+
from .validation import fail
|
|
32
|
+
|
|
33
|
+
#: The largest OpenAPI document the importer reads. GitHub's is ~13 MB.
|
|
34
|
+
MAX_BYTES = 64 * 1024 * 1024
|
|
35
|
+
MAX_REDIRECTS = 5
|
|
36
|
+
FETCH_TIMEOUT_S = 30
|
|
37
|
+
LOCATION = "/openapi/source"
|
|
38
|
+
_PRIVATE_HOSTNAME = re.compile(r"^(localhost$|.*\.local$|.*\.internal$|metadata\.google\.internal$)", re.IGNORECASE)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def public_address(host: str) -> None:
|
|
42
|
+
"""Refuse a host that is, or resolves to, something the network around the
|
|
43
|
+
command must not reach: loopback, RFC1918, link-local and cloud metadata,
|
|
44
|
+
CGNAT, v6 ULA/link-local, ``.local``/``.internal`` names."""
|
|
45
|
+
if _PRIVATE_HOSTNAME.match(host):
|
|
46
|
+
fail("source", f"Refusing to fetch a private/internal host: {host}", LOCATION)
|
|
47
|
+
try:
|
|
48
|
+
literal = ipaddress.ip_address(host)
|
|
49
|
+
except ValueError:
|
|
50
|
+
literal = None
|
|
51
|
+
if literal is not None and _is_private(literal):
|
|
52
|
+
fail("source", f"Refusing to fetch a private address: {host}", LOCATION)
|
|
53
|
+
if literal is not None:
|
|
54
|
+
return
|
|
55
|
+
try:
|
|
56
|
+
infos = socket.getaddrinfo(host, None)
|
|
57
|
+
except OSError:
|
|
58
|
+
infos = []
|
|
59
|
+
if not infos:
|
|
60
|
+
fail("source", f"Could not resolve host: {host}", LOCATION)
|
|
61
|
+
for info in infos:
|
|
62
|
+
address = ipaddress.ip_address(info[4][0])
|
|
63
|
+
if _is_private(address):
|
|
64
|
+
fail("source", f"Refusing to fetch {host}: it resolves to a private address ({address})", LOCATION)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_private(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
|
68
|
+
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
|
|
69
|
+
address = address.ipv4_mapped
|
|
70
|
+
if isinstance(address, ipaddress.IPv4Address):
|
|
71
|
+
cgnat = ipaddress.ip_network("100.64.0.0/10")
|
|
72
|
+
return (
|
|
73
|
+
address.is_private
|
|
74
|
+
or address.is_loopback
|
|
75
|
+
or address.is_link_local
|
|
76
|
+
or address.is_unspecified
|
|
77
|
+
or address in cgnat
|
|
78
|
+
)
|
|
79
|
+
return address.is_private or address.is_loopback or address.is_link_local or address.is_unspecified
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
83
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
_OPENER = urllib.request.build_opener(_NoRedirect)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _read_capped(response, url: str) -> bytes:
|
|
91
|
+
declared = response.headers.get("Content-Length")
|
|
92
|
+
if declared and declared.isdigit() and int(declared) > MAX_BYTES:
|
|
93
|
+
fail("source", f"{url} is {declared} bytes, over the {MAX_BYTES}-byte limit", LOCATION, limit=MAX_BYTES)
|
|
94
|
+
chunks: list[bytes] = []
|
|
95
|
+
received = 0
|
|
96
|
+
while True:
|
|
97
|
+
chunk = response.read(1024 * 1024)
|
|
98
|
+
if not chunk:
|
|
99
|
+
return b"".join(chunks)
|
|
100
|
+
received += len(chunk)
|
|
101
|
+
if received > MAX_BYTES:
|
|
102
|
+
fail("source", f"{url} exceeded the {MAX_BYTES}-byte limit mid-stream", LOCATION, limit=MAX_BYTES)
|
|
103
|
+
chunks.append(chunk)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def fetch_public(url: str) -> tuple[bytes, str]:
|
|
107
|
+
"""The body of a public http(s) URL and the URL that finally answered."""
|
|
108
|
+
for _hop in range(MAX_REDIRECTS + 1):
|
|
109
|
+
parts = urllib.parse.urlsplit(url)
|
|
110
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
111
|
+
fail("source", f"{url} is not an http(s) URL", LOCATION)
|
|
112
|
+
public_address(parts.hostname)
|
|
113
|
+
request = urllib.request.Request(url, headers={"User-Agent": "gateway-worlds/1 (schema init --from-openapi)"})
|
|
114
|
+
try:
|
|
115
|
+
with _OPENER.open(request, timeout=FETCH_TIMEOUT_S) as response:
|
|
116
|
+
return _read_capped(response, url), url
|
|
117
|
+
except urllib.error.HTTPError as error:
|
|
118
|
+
if 300 <= error.code < 400 and error.headers.get("Location"):
|
|
119
|
+
url = urllib.parse.urljoin(url, error.headers["Location"])
|
|
120
|
+
continue
|
|
121
|
+
fail("source", f"{url} answered HTTP {error.code}", LOCATION, status=error.code)
|
|
122
|
+
except (urllib.error.URLError, OSError) as error:
|
|
123
|
+
fail("source", f"{url} could not be fetched: {error}", LOCATION)
|
|
124
|
+
fail("source", f"Too many redirects fetching {url} (limit {MAX_REDIRECTS})", LOCATION)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def public_label(url: str) -> str:
|
|
128
|
+
"""The URL with userinfo and query string removed: what provenance records."""
|
|
129
|
+
parts = urllib.parse.urlsplit(url)
|
|
130
|
+
host = parts.hostname or ""
|
|
131
|
+
if parts.port:
|
|
132
|
+
host = f"{host}:{parts.port}"
|
|
133
|
+
return urllib.parse.urlunsplit((parts.scheme, host, parts.path, "", ""))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
_HTTP_URL = re.compile(r"^https?://", re.IGNORECASE)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class OpenApiSource:
|
|
141
|
+
"""One OpenAPI document as read: its text, where it came from and its hash."""
|
|
142
|
+
|
|
143
|
+
kind: str # local | http | drive | text
|
|
144
|
+
label: str
|
|
145
|
+
text: str
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def sha256(self) -> str:
|
|
149
|
+
return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def size(self) -> int:
|
|
153
|
+
return len(self.text.encode("utf-8"))
|
|
154
|
+
|
|
155
|
+
def document(self) -> dict:
|
|
156
|
+
return parse_openapi(self.text, self.label, location=LOCATION)
|
|
157
|
+
|
|
158
|
+
@staticmethod
|
|
159
|
+
def kind_of(source: str) -> str:
|
|
160
|
+
"""How a spelled source is read: ``drive`` for ``drive:<path>``, ``http`` for a URL, else ``local``."""
|
|
161
|
+
if source.startswith("drive:"):
|
|
162
|
+
return "drive"
|
|
163
|
+
return "http" if _HTTP_URL.match(source) else "local"
|
|
164
|
+
|
|
165
|
+
@staticmethod
|
|
166
|
+
def safe_label(source: str) -> str:
|
|
167
|
+
"""What provenance records for a spelled source: a URL loses its userinfo and query string."""
|
|
168
|
+
return public_label(source) if _HTTP_URL.match(source) else source
|
|
169
|
+
|
|
170
|
+
@classmethod
|
|
171
|
+
def from_bytes(cls, data: bytes, label: str, kind: str) -> OpenApiSource:
|
|
172
|
+
if len(data) > MAX_BYTES:
|
|
173
|
+
fail("source", f"{label} is {len(data)} bytes, over the {MAX_BYTES}-byte limit", LOCATION, limit=MAX_BYTES)
|
|
174
|
+
try:
|
|
175
|
+
text = data.decode("utf-8")
|
|
176
|
+
except UnicodeDecodeError:
|
|
177
|
+
fail("source", f"{label} is not UTF-8 text", LOCATION)
|
|
178
|
+
return cls(kind, label, text)
|
|
179
|
+
|
|
180
|
+
@classmethod
|
|
181
|
+
def from_text(cls, text: str, label: str, kind: str = "text") -> OpenApiSource:
|
|
182
|
+
return cls.from_bytes(text.encode("utf-8"), label, kind)
|
|
183
|
+
|
|
184
|
+
@classmethod
|
|
185
|
+
def read(cls, source: str) -> OpenApiSource:
|
|
186
|
+
"""A local path or a public URL. ``drive:<path>`` is the platform CLI's to
|
|
187
|
+
stage (it needs the project's credentials, which the sandbox does not hold)."""
|
|
188
|
+
if source.startswith("drive:"):
|
|
189
|
+
fail(
|
|
190
|
+
"source",
|
|
191
|
+
"drive:<path> is read through the platform: run `gateway worlds schema init <dir> --from-openapi "
|
|
192
|
+
f"{source}` from the platform CLI, or pass the file's local path here",
|
|
193
|
+
LOCATION,
|
|
194
|
+
)
|
|
195
|
+
if _HTTP_URL.match(source):
|
|
196
|
+
data, final_url = fetch_public(source)
|
|
197
|
+
return cls.from_bytes(data, public_label(final_url), "http")
|
|
198
|
+
if re.match(r"^[a-z][a-z0-9+.-]*://", source, re.IGNORECASE):
|
|
199
|
+
fail("source", f"{source}: only local paths, drive:<path> and http(s) URLs are supported", LOCATION)
|
|
200
|
+
path = Path(source)
|
|
201
|
+
if not path.is_file():
|
|
202
|
+
fail("source", f"{source} is not a file", LOCATION)
|
|
203
|
+
if path.stat().st_size > MAX_BYTES:
|
|
204
|
+
fail(
|
|
205
|
+
"source",
|
|
206
|
+
f"{source} is {path.stat().st_size} bytes, over the {MAX_BYTES}-byte limit",
|
|
207
|
+
LOCATION,
|
|
208
|
+
limit=MAX_BYTES,
|
|
209
|
+
)
|
|
210
|
+
return cls.from_bytes(path.read_bytes(), str(path), "local")
|