@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""The world versions a World Host has cached (:class:`VersionCache`), each one
|
|
2
|
+
:class:`CachedVersion` built once from its bundle and shared by every session of it."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import tarfile
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
import urllib.request
|
|
12
|
+
import uuid
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from .._tar import safe_tar_extract
|
|
18
|
+
from ..world import WorldTree
|
|
19
|
+
from . import snapshot as snapshots
|
|
20
|
+
from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
|
|
21
|
+
from .compiler import check_bundle
|
|
22
|
+
from .connector import Spec, load_spec
|
|
23
|
+
from .database import WorldDatabase
|
|
24
|
+
from .dependencies import Dependency
|
|
25
|
+
from .door import Door
|
|
26
|
+
from .host_surfaces import UiSurface, load_ui_surface
|
|
27
|
+
from .host_transfers import TRANSFER_TIMEOUT_S
|
|
28
|
+
from .lanes import Lanes
|
|
29
|
+
from .runtime import SnapshotRuntime, TreeRuntime, WorldRuntime
|
|
30
|
+
from .served_names import ServedNames
|
|
31
|
+
from .tools_world import DependencyWorld, ToolsVersion, is_tools_bundle, load_tools_version
|
|
32
|
+
from .validation import fail
|
|
33
|
+
|
|
34
|
+
BUNDLED_WORLD = "server/world"
|
|
35
|
+
WORLD_SPEC = "schema/world.json"
|
|
36
|
+
#: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
|
|
37
|
+
DEFAULT_MAX_BUNDLE_MB = 2048
|
|
38
|
+
MAX_BUNDLE_MEMBERS = 200_000
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def world_tree(bundle: Path) -> Path:
|
|
42
|
+
"""The compiled schema world inside an extracted bundle (``WorldTree.world``:
|
|
43
|
+
``bench push`` packages a schema world as its api bundle with the tree
|
|
44
|
+
under ``server/world``; a bare tree is its own root). The host opens
|
|
45
|
+
compiled worlds only, so a tree without ``schema/world.json`` refuses."""
|
|
46
|
+
tree = WorldTree.from_dir(bundle)
|
|
47
|
+
if tree.has(tree.world_path(WORLD_SPEC)):
|
|
48
|
+
return tree.world
|
|
49
|
+
fail(
|
|
50
|
+
"host",
|
|
51
|
+
f"{bundle} holds no schema world the host can open (no {BUNDLED_WORLD}/{WORLD_SPEC} "
|
|
52
|
+
f"nor {WORLD_SPEC}); the host serves schema worlds and api bundles only",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class CachedVersion:
|
|
58
|
+
"""One cached version, of either shape the host serves: an api bundle (a
|
|
59
|
+
connector ``spec`` and ``settings``, sessions cloned from ``snapshot``) or a
|
|
60
|
+
schema world stored as itself (``tools``: handlers, tasks and verifiers,
|
|
61
|
+
sessions copied from its template — and ``spec`` + ``settings`` too when
|
|
62
|
+
its tree carries connector.toml, the HTTP surface the same session serves
|
|
63
|
+
over the same store). ``snapshot`` and ``served`` are an api bundle's."""
|
|
64
|
+
|
|
65
|
+
version_id: str
|
|
66
|
+
bundle: Path
|
|
67
|
+
tree: Path
|
|
68
|
+
spec: Spec | None
|
|
69
|
+
settings: ApiSettings | None
|
|
70
|
+
snapshot: Path | None
|
|
71
|
+
engine: type[WorldDatabase]
|
|
72
|
+
#: None in the supervisor: only a worker imports world-authored code.
|
|
73
|
+
handlers: dict[str, Handler] | None = None
|
|
74
|
+
ui: UiSurface | None = None
|
|
75
|
+
tools: ToolsVersion | None = None
|
|
76
|
+
served: snapshots.ServedSnapshot | None = None
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def runtime_class(self) -> type[WorldRuntime]:
|
|
80
|
+
"""The one place a session's engine is chosen: a schema world runs as
|
|
81
|
+
the tree it is; an api bundle on the snapshot READ PATH that step 6
|
|
82
|
+
deletes with the bundle shape itself."""
|
|
83
|
+
return TreeRuntime if self.tools is not None else SnapshotRuntime
|
|
84
|
+
|
|
85
|
+
def router(self) -> Router:
|
|
86
|
+
"""The version's routes, for one session's world."""
|
|
87
|
+
assert self.spec is not None and self.settings is not None
|
|
88
|
+
return Router(self.spec, self.settings)
|
|
89
|
+
|
|
90
|
+
def door(self, token: str | None, clock: Callable[[], float] = time.time) -> Door | None:
|
|
91
|
+
"""The door of a session's routes for its ``token``, None for a world with none."""
|
|
92
|
+
if self.spec is None or self.settings is None:
|
|
93
|
+
return None
|
|
94
|
+
return Door.of(self.spec, self.settings, token, clock)
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def authored_code(self) -> str | None:
|
|
98
|
+
"""What world-authored code the version runs, None for none: only a
|
|
99
|
+
worker may open a version that runs any."""
|
|
100
|
+
return self.runtime_class.authored_code(self)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def needs_worker(self) -> bool:
|
|
104
|
+
return self.authored_code is not None
|
|
105
|
+
|
|
106
|
+
def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
|
|
107
|
+
"""The tools a session task may allow, beside the routes it may not."""
|
|
108
|
+
return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def declared_dependencies(self) -> dict[str, Dependency]:
|
|
112
|
+
return self.runtime_class.declared_dependencies(self)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class VersionCache:
|
|
116
|
+
"""Every world version this host has seen, on local disk. The first session
|
|
117
|
+
of a version pays for the download and extract; the rest clone its
|
|
118
|
+
snapshot. Nothing here is per session, so a hot version is cheap to serve
|
|
119
|
+
many times over."""
|
|
120
|
+
|
|
121
|
+
def __init__(self, root: Path, *, run_handlers: bool = False, max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB):
|
|
122
|
+
self.root = Path(root)
|
|
123
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
#: Off in the supervisor: only a version worker imports world-authored code.
|
|
125
|
+
self.run_handlers = run_handlers
|
|
126
|
+
self.max_bundle_bytes = max_bundle_mb * (1 << 20)
|
|
127
|
+
self._lock = threading.Lock()
|
|
128
|
+
self._versions: dict[str, CachedVersion] = {}
|
|
129
|
+
self._builds = Lanes()
|
|
130
|
+
|
|
131
|
+
def ensure(
|
|
132
|
+
self,
|
|
133
|
+
version_id: str,
|
|
134
|
+
bundle_url: str,
|
|
135
|
+
snapshot: dict[str, str] | None,
|
|
136
|
+
) -> CachedVersion:
|
|
137
|
+
hit = self._cached(version_id)
|
|
138
|
+
if hit is not None:
|
|
139
|
+
return hit
|
|
140
|
+
with self._builds.turn(version_id):
|
|
141
|
+
return self._cached(version_id) or self._store(self._build(version_id, bundle_url, snapshot))
|
|
142
|
+
|
|
143
|
+
def _cached(self, version_id: str) -> CachedVersion | None:
|
|
144
|
+
with self._lock:
|
|
145
|
+
return self._versions.get(version_id)
|
|
146
|
+
|
|
147
|
+
def _store(self, cached: CachedVersion) -> CachedVersion:
|
|
148
|
+
with self._lock:
|
|
149
|
+
self._versions[cached.version_id] = cached
|
|
150
|
+
return cached
|
|
151
|
+
|
|
152
|
+
def has(self, version_id: str) -> bool:
|
|
153
|
+
with self._lock:
|
|
154
|
+
return version_id in self._versions
|
|
155
|
+
|
|
156
|
+
def version_ids(self) -> list[str]:
|
|
157
|
+
with self._lock:
|
|
158
|
+
return sorted(self._versions)
|
|
159
|
+
|
|
160
|
+
def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
|
|
161
|
+
bundle = self.root / version_id / "bundle"
|
|
162
|
+
if not bundle.is_dir():
|
|
163
|
+
self._download(bundle_url, bundle)
|
|
164
|
+
if is_tools_bundle(bundle):
|
|
165
|
+
tools = load_tools_version(
|
|
166
|
+
bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
|
|
167
|
+
)
|
|
168
|
+
return CachedVersion(
|
|
169
|
+
version_id=version_id,
|
|
170
|
+
bundle=bundle,
|
|
171
|
+
tree=bundle,
|
|
172
|
+
spec=tools.spec,
|
|
173
|
+
settings=tools.settings,
|
|
174
|
+
snapshot=None,
|
|
175
|
+
engine=tools.engine,
|
|
176
|
+
handlers=tools.api_handlers,
|
|
177
|
+
ui=load_ui_surface(bundle) if tools.spec is not None else None,
|
|
178
|
+
tools=tools,
|
|
179
|
+
)
|
|
180
|
+
tree = world_tree(bundle)
|
|
181
|
+
contract = check_bundle(tree)
|
|
182
|
+
if snapshot:
|
|
183
|
+
snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
184
|
+
served = snapshots.ServedSnapshot.of(tree)
|
|
185
|
+
spec = load_spec(tree)
|
|
186
|
+
settings = load_api_settings(tree, spec)
|
|
187
|
+
handlers = load_handlers(settings) if self.run_handlers and settings.handlers is not None else None
|
|
188
|
+
snap = snapshots.default_snapshot(tree, contract)
|
|
189
|
+
if not snap.is_file():
|
|
190
|
+
fail(
|
|
191
|
+
"host",
|
|
192
|
+
f"version {version_id} has no compiled snapshot ({snap.relative_to(tree)}); "
|
|
193
|
+
"publish a data snapshot or pin one on the version",
|
|
194
|
+
)
|
|
195
|
+
return CachedVersion(
|
|
196
|
+
version_id=version_id,
|
|
197
|
+
bundle=bundle,
|
|
198
|
+
tree=tree,
|
|
199
|
+
spec=spec,
|
|
200
|
+
settings=settings,
|
|
201
|
+
snapshot=snap,
|
|
202
|
+
engine=WorldDatabase.for_contract(contract),
|
|
203
|
+
served=served,
|
|
204
|
+
handlers=handlers,
|
|
205
|
+
ui=load_ui_surface(bundle),
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
def _download(self, url: str, dest: Path) -> None:
|
|
209
|
+
"""Fetch and extract into a sibling staging directory, then rename it into
|
|
210
|
+
place: ``dest`` exists only once it is complete, so a download that dies
|
|
211
|
+
halfway is never mistaken for a cached bundle on the next open."""
|
|
212
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
213
|
+
staging = dest.parent / f".{dest.name}.{uuid.uuid4().hex}"
|
|
214
|
+
tgz = staging.with_suffix(".tar.gz")
|
|
215
|
+
limit = self.max_bundle_bytes
|
|
216
|
+
try:
|
|
217
|
+
try:
|
|
218
|
+
with (
|
|
219
|
+
urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
|
|
220
|
+
open(tgz, "wb") as out,
|
|
221
|
+
):
|
|
222
|
+
copied = 0
|
|
223
|
+
for chunk in iter(lambda: r.read(1 << 20), b""):
|
|
224
|
+
copied += len(chunk)
|
|
225
|
+
if copied > limit:
|
|
226
|
+
raise ValueError(f"the bundle is larger than {limit} bytes")
|
|
227
|
+
out.write(chunk)
|
|
228
|
+
except Exception as error: # noqa: BLE001 - name the stage in the message
|
|
229
|
+
fail("host", f"could not download the bundle: {error}")
|
|
230
|
+
staging.mkdir()
|
|
231
|
+
with tarfile.open(tgz, "r:gz") as t:
|
|
232
|
+
members = t.getmembers()
|
|
233
|
+
if len(members) > MAX_BUNDLE_MEMBERS:
|
|
234
|
+
fail("host", f"the bundle holds {len(members)} files, more than the {MAX_BUNDLE_MEMBERS} allowed")
|
|
235
|
+
extracted = sum(m.size for m in members)
|
|
236
|
+
if extracted > limit:
|
|
237
|
+
fail("host", f"the bundle extracts to {extracted} bytes, more than the {limit} allowed")
|
|
238
|
+
safe_tar_extract(t, staging)
|
|
239
|
+
os.replace(staging, dest)
|
|
240
|
+
finally:
|
|
241
|
+
tgz.unlink(missing_ok=True)
|
|
242
|
+
shutil.rmtree(staging, ignore_errors=True)
|