@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -0,0 +1,242 @@
1
+ """The world versions a World Host has cached (:class:`VersionCache`), each one
2
+ :class:`CachedVersion` built once from its bundle and shared by every session of it."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import os
7
+ import shutil
8
+ import tarfile
9
+ import threading
10
+ import time
11
+ import urllib.request
12
+ import uuid
13
+ from collections.abc import Callable
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+
17
+ from .._tar import safe_tar_extract
18
+ from ..world import WorldTree
19
+ from . import snapshot as snapshots
20
+ from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
21
+ from .compiler import check_bundle
22
+ from .connector import Spec, load_spec
23
+ from .database import WorldDatabase
24
+ from .dependencies import Dependency
25
+ from .door import Door
26
+ from .host_surfaces import UiSurface, load_ui_surface
27
+ from .host_transfers import TRANSFER_TIMEOUT_S
28
+ from .lanes import Lanes
29
+ from .runtime import SnapshotRuntime, TreeRuntime, WorldRuntime
30
+ from .served_names import ServedNames
31
+ from .tools_world import DependencyWorld, ToolsVersion, is_tools_bundle, load_tools_version
32
+ from .validation import fail
33
+
34
+ BUNDLED_WORLD = "server/world"
35
+ WORLD_SPEC = "schema/world.json"
36
+ #: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
37
+ DEFAULT_MAX_BUNDLE_MB = 2048
38
+ MAX_BUNDLE_MEMBERS = 200_000
39
+
40
+
41
+ def world_tree(bundle: Path) -> Path:
42
+ """The compiled schema world inside an extracted bundle (``WorldTree.world``:
43
+ ``bench push`` packages a schema world as its api bundle with the tree
44
+ under ``server/world``; a bare tree is its own root). The host opens
45
+ compiled worlds only, so a tree without ``schema/world.json`` refuses."""
46
+ tree = WorldTree.from_dir(bundle)
47
+ if tree.has(tree.world_path(WORLD_SPEC)):
48
+ return tree.world
49
+ fail(
50
+ "host",
51
+ f"{bundle} holds no schema world the host can open (no {BUNDLED_WORLD}/{WORLD_SPEC} "
52
+ f"nor {WORLD_SPEC}); the host serves schema worlds and api bundles only",
53
+ )
54
+
55
+
56
+ @dataclass
57
+ class CachedVersion:
58
+ """One cached version, of either shape the host serves: an api bundle (a
59
+ connector ``spec`` and ``settings``, sessions cloned from ``snapshot``) or a
60
+ schema world stored as itself (``tools``: handlers, tasks and verifiers,
61
+ sessions copied from its template — and ``spec`` + ``settings`` too when
62
+ its tree carries connector.toml, the HTTP surface the same session serves
63
+ over the same store). ``snapshot`` and ``served`` are an api bundle's."""
64
+
65
+ version_id: str
66
+ bundle: Path
67
+ tree: Path
68
+ spec: Spec | None
69
+ settings: ApiSettings | None
70
+ snapshot: Path | None
71
+ engine: type[WorldDatabase]
72
+ #: None in the supervisor: only a worker imports world-authored code.
73
+ handlers: dict[str, Handler] | None = None
74
+ ui: UiSurface | None = None
75
+ tools: ToolsVersion | None = None
76
+ served: snapshots.ServedSnapshot | None = None
77
+
78
+ @property
79
+ def runtime_class(self) -> type[WorldRuntime]:
80
+ """The one place a session's engine is chosen: a schema world runs as
81
+ the tree it is; an api bundle on the snapshot READ PATH that step 6
82
+ deletes with the bundle shape itself."""
83
+ return TreeRuntime if self.tools is not None else SnapshotRuntime
84
+
85
+ def router(self) -> Router:
86
+ """The version's routes, for one session's world."""
87
+ assert self.spec is not None and self.settings is not None
88
+ return Router(self.spec, self.settings)
89
+
90
+ def door(self, token: str | None, clock: Callable[[], float] = time.time) -> Door | None:
91
+ """The door of a session's routes for its ``token``, None for a world with none."""
92
+ if self.spec is None or self.settings is None:
93
+ return None
94
+ return Door.of(self.spec, self.settings, token, clock)
95
+
96
+ @property
97
+ def authored_code(self) -> str | None:
98
+ """What world-authored code the version runs, None for none: only a
99
+ worker may open a version that runs any."""
100
+ return self.runtime_class.authored_code(self)
101
+
102
+ @property
103
+ def needs_worker(self) -> bool:
104
+ return self.authored_code is not None
105
+
106
+ def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
107
+ """The tools a session task may allow, beside the routes it may not."""
108
+ return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
109
+
110
+ @property
111
+ def declared_dependencies(self) -> dict[str, Dependency]:
112
+ return self.runtime_class.declared_dependencies(self)
113
+
114
+
115
+ class VersionCache:
116
+ """Every world version this host has seen, on local disk. The first session
117
+ of a version pays for the download and extract; the rest clone its
118
+ snapshot. Nothing here is per session, so a hot version is cheap to serve
119
+ many times over."""
120
+
121
+ def __init__(self, root: Path, *, run_handlers: bool = False, max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB):
122
+ self.root = Path(root)
123
+ self.root.mkdir(parents=True, exist_ok=True)
124
+ #: Off in the supervisor: only a version worker imports world-authored code.
125
+ self.run_handlers = run_handlers
126
+ self.max_bundle_bytes = max_bundle_mb * (1 << 20)
127
+ self._lock = threading.Lock()
128
+ self._versions: dict[str, CachedVersion] = {}
129
+ self._builds = Lanes()
130
+
131
+ def ensure(
132
+ self,
133
+ version_id: str,
134
+ bundle_url: str,
135
+ snapshot: dict[str, str] | None,
136
+ ) -> CachedVersion:
137
+ hit = self._cached(version_id)
138
+ if hit is not None:
139
+ return hit
140
+ with self._builds.turn(version_id):
141
+ return self._cached(version_id) or self._store(self._build(version_id, bundle_url, snapshot))
142
+
143
+ def _cached(self, version_id: str) -> CachedVersion | None:
144
+ with self._lock:
145
+ return self._versions.get(version_id)
146
+
147
+ def _store(self, cached: CachedVersion) -> CachedVersion:
148
+ with self._lock:
149
+ self._versions[cached.version_id] = cached
150
+ return cached
151
+
152
+ def has(self, version_id: str) -> bool:
153
+ with self._lock:
154
+ return version_id in self._versions
155
+
156
+ def version_ids(self) -> list[str]:
157
+ with self._lock:
158
+ return sorted(self._versions)
159
+
160
+ def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
161
+ bundle = self.root / version_id / "bundle"
162
+ if not bundle.is_dir():
163
+ self._download(bundle_url, bundle)
164
+ if is_tools_bundle(bundle):
165
+ tools = load_tools_version(
166
+ bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
167
+ )
168
+ return CachedVersion(
169
+ version_id=version_id,
170
+ bundle=bundle,
171
+ tree=bundle,
172
+ spec=tools.spec,
173
+ settings=tools.settings,
174
+ snapshot=None,
175
+ engine=tools.engine,
176
+ handlers=tools.api_handlers,
177
+ ui=load_ui_surface(bundle) if tools.spec is not None else None,
178
+ tools=tools,
179
+ )
180
+ tree = world_tree(bundle)
181
+ contract = check_bundle(tree)
182
+ if snapshot:
183
+ snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
184
+ served = snapshots.ServedSnapshot.of(tree)
185
+ spec = load_spec(tree)
186
+ settings = load_api_settings(tree, spec)
187
+ handlers = load_handlers(settings) if self.run_handlers and settings.handlers is not None else None
188
+ snap = snapshots.default_snapshot(tree, contract)
189
+ if not snap.is_file():
190
+ fail(
191
+ "host",
192
+ f"version {version_id} has no compiled snapshot ({snap.relative_to(tree)}); "
193
+ "publish a data snapshot or pin one on the version",
194
+ )
195
+ return CachedVersion(
196
+ version_id=version_id,
197
+ bundle=bundle,
198
+ tree=tree,
199
+ spec=spec,
200
+ settings=settings,
201
+ snapshot=snap,
202
+ engine=WorldDatabase.for_contract(contract),
203
+ served=served,
204
+ handlers=handlers,
205
+ ui=load_ui_surface(bundle),
206
+ )
207
+
208
+ def _download(self, url: str, dest: Path) -> None:
209
+ """Fetch and extract into a sibling staging directory, then rename it into
210
+ place: ``dest`` exists only once it is complete, so a download that dies
211
+ halfway is never mistaken for a cached bundle on the next open."""
212
+ dest.parent.mkdir(parents=True, exist_ok=True)
213
+ staging = dest.parent / f".{dest.name}.{uuid.uuid4().hex}"
214
+ tgz = staging.with_suffix(".tar.gz")
215
+ limit = self.max_bundle_bytes
216
+ try:
217
+ try:
218
+ with (
219
+ urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
220
+ open(tgz, "wb") as out,
221
+ ):
222
+ copied = 0
223
+ for chunk in iter(lambda: r.read(1 << 20), b""):
224
+ copied += len(chunk)
225
+ if copied > limit:
226
+ raise ValueError(f"the bundle is larger than {limit} bytes")
227
+ out.write(chunk)
228
+ except Exception as error: # noqa: BLE001 - name the stage in the message
229
+ fail("host", f"could not download the bundle: {error}")
230
+ staging.mkdir()
231
+ with tarfile.open(tgz, "r:gz") as t:
232
+ members = t.getmembers()
233
+ if len(members) > MAX_BUNDLE_MEMBERS:
234
+ fail("host", f"the bundle holds {len(members)} files, more than the {MAX_BUNDLE_MEMBERS} allowed")
235
+ extracted = sum(m.size for m in members)
236
+ if extracted > limit:
237
+ fail("host", f"the bundle extracts to {extracted} bytes, more than the {limit} allowed")
238
+ safe_tar_extract(t, staging)
239
+ os.replace(staging, dest)
240
+ finally:
241
+ tgz.unlink(missing_ok=True)
242
+ shutil.rmtree(staging, ignore_errors=True)