@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -5,26 +5,54 @@ starts from. Its rows are proven once, when the host caches the version; an
|
|
|
5
5
|
open or a reset clones it and the store runs only its contract and DDL check.
|
|
6
6
|
A :class:`SessionExport` is a checkpoint a session continues from: its bytes
|
|
7
7
|
must hash to the sha256 the platform recorded when the host exported it, and
|
|
8
|
-
it carries the session's call log beside the rows.
|
|
8
|
+
it carries the session's call log beside the rows. A :class:`RecoveredSession`
|
|
9
|
+
is the directory a version worker that died left a session in, set aside by its
|
|
10
|
+
supervisor: its store's committed state and its call journal are the checkpoint
|
|
11
|
+
the next worker continues the session from.
|
|
9
12
|
"""
|
|
10
13
|
|
|
11
14
|
from __future__ import annotations
|
|
12
15
|
|
|
13
16
|
import json
|
|
14
17
|
import os
|
|
18
|
+
import shutil
|
|
15
19
|
import uuid
|
|
16
20
|
from pathlib import Path
|
|
17
|
-
from typing import NamedTuple
|
|
21
|
+
from typing import TYPE_CHECKING, Iterable, NamedTuple
|
|
18
22
|
|
|
19
23
|
from .._materialize import materialize_template
|
|
24
|
+
from .call_log import CallJournal
|
|
20
25
|
from .compiler import Contract
|
|
21
26
|
from .database import WorldDatabase
|
|
27
|
+
from .database.reads import FORMAT as READ_INDEXES
|
|
22
28
|
from .snapshot import contract_hash, file_sha256, runtime_version
|
|
23
29
|
from .validation import fail
|
|
24
30
|
|
|
31
|
+
if TYPE_CHECKING:
|
|
32
|
+
from .runtime import WorldRuntime
|
|
33
|
+
|
|
34
|
+
#: A session's call journal, beside its store in the session's directory.
|
|
35
|
+
CALLS_JOURNAL = "calls.jsonl"
|
|
36
|
+
#: A session's unsent request ledger entries, beside its store (:class:`~.request_ledger.RequestOutbox`).
|
|
37
|
+
LEDGER_JOURNAL = "requests.jsonl"
|
|
38
|
+
#: Where a supervisor sets a dead worker's session directories aside, under that worker's home.
|
|
39
|
+
RECOVERED_DIR = "recovered"
|
|
40
|
+
|
|
25
41
|
_PROVEN = object()
|
|
26
42
|
|
|
27
43
|
|
|
44
|
+
def real_dir(path: Path) -> bool:
|
|
45
|
+
"""A directory at ``path`` itself, not a link to one somewhere else."""
|
|
46
|
+
return path.is_dir() and not path.is_symlink()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def clear(path: Path) -> None:
|
|
50
|
+
"""Remove whatever sits at ``path``, a symlink as a link, never what it points at."""
|
|
51
|
+
if path.is_symlink() or (path.exists() and not path.is_dir()):
|
|
52
|
+
path.unlink()
|
|
53
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
54
|
+
|
|
55
|
+
|
|
28
56
|
def refuse_digest_mismatch(path: Path, sha256: str) -> None:
|
|
29
57
|
"""Refuse a session export whose bytes do not hash to the sha256 recorded for them."""
|
|
30
58
|
actual = file_sha256(path)
|
|
@@ -68,12 +96,21 @@ class ProvenTemplate:
|
|
|
68
96
|
return template
|
|
69
97
|
|
|
70
98
|
@classmethod
|
|
71
|
-
def prove(
|
|
72
|
-
|
|
99
|
+
def prove(
|
|
100
|
+
cls,
|
|
101
|
+
bundle: Path,
|
|
102
|
+
path: Path,
|
|
103
|
+
contract: Contract,
|
|
104
|
+
*,
|
|
105
|
+
fetched: bool,
|
|
106
|
+
read_fields: Iterable[tuple[str, tuple[str, ...]]] = (),
|
|
107
|
+
) -> ProvenTemplate:
|
|
108
|
+
"""The version's template at ``path``, with its read indexes (its keys' and the
|
|
109
|
+
``read_fields`` its routes filter on). ``fetched`` says the tree's snapshot is
|
|
73
110
|
the platform's, downloaded and verified against its recorded sha256 just now."""
|
|
74
111
|
template = cls(path, contract, _PROVEN)
|
|
75
112
|
if not template._recorded():
|
|
76
|
-
template._build(bundle, fetched=fetched)
|
|
113
|
+
template._build(bundle, fetched=fetched, read_fields=read_fields)
|
|
77
114
|
return template
|
|
78
115
|
|
|
79
116
|
@property
|
|
@@ -85,6 +122,7 @@ class ProvenTemplate:
|
|
|
85
122
|
"sha256": file_sha256(self.path),
|
|
86
123
|
"contractHash": contract_hash(self.contract),
|
|
87
124
|
"runtime": runtime_version(),
|
|
125
|
+
"readIndexes": READ_INDEXES,
|
|
88
126
|
}
|
|
89
127
|
|
|
90
128
|
def _recorded(self) -> bool:
|
|
@@ -94,12 +132,12 @@ class ProvenTemplate:
|
|
|
94
132
|
return False
|
|
95
133
|
return self.path.is_file() and recorded == self._proof()
|
|
96
134
|
|
|
97
|
-
def _build(self, bundle: Path, *, fetched: bool) -> None:
|
|
135
|
+
def _build(self, bundle: Path, *, fetched: bool, read_fields: Iterable[tuple[str, tuple[str, ...]]]) -> None:
|
|
98
136
|
staging = self.path.with_name(f".{self.path.name}.{uuid.uuid4().hex}")
|
|
99
137
|
try:
|
|
100
138
|
copied = materialize_template(bundle, staging, require_sql=True) is not None
|
|
101
|
-
|
|
102
|
-
|
|
139
|
+
with WorldDatabase.open(staging, self.contract, audit=copied and not fetched) as store:
|
|
140
|
+
store.index_reads(read_fields)
|
|
103
141
|
os.replace(staging, self.path)
|
|
104
142
|
finally:
|
|
105
143
|
staging.unlink(missing_ok=True)
|
|
@@ -117,28 +155,68 @@ class ExportFile(NamedTuple):
|
|
|
117
155
|
def for_worker(self) -> dict[str, str]:
|
|
118
156
|
return {"path": str(self.path), "sha256": self.sha256}
|
|
119
157
|
|
|
158
|
+
def checkpoint(self, engine: type[WorldDatabase], runtime: type[WorldRuntime]) -> SessionExport:
|
|
159
|
+
return SessionExport.verified(self, engine)
|
|
120
160
|
|
|
121
|
-
|
|
161
|
+
def discard(self) -> None:
|
|
162
|
+
self.path.unlink(missing_ok=True)
|
|
122
163
|
|
|
123
164
|
|
|
124
|
-
class
|
|
125
|
-
"""A
|
|
165
|
+
class RecoveredSession(NamedTuple):
|
|
166
|
+
"""A session's directory a dead version worker left on this host, which its
|
|
167
|
+
supervisor set aside before starting the next worker (that worker clears the
|
|
168
|
+
sessions directory it starts in). The file was the session's own store and the
|
|
169
|
+
worker that wrote it ran this version's code, as the next one does: it is the
|
|
170
|
+
session's committed state, every row audited as it committed, and its call
|
|
171
|
+
journal beside it."""
|
|
172
|
+
|
|
173
|
+
directory: Path
|
|
174
|
+
|
|
175
|
+
def checkpoint(self, engine: type[WorldDatabase], runtime: type[WorldRuntime]) -> SessionExport:
|
|
176
|
+
directory = self.directory
|
|
177
|
+
return SessionExport.recovered_from(
|
|
178
|
+
directory, engine, runtime.live_state(directory), runtime.dependency_states(directory)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def discard(self) -> None:
|
|
182
|
+
shutil.rmtree(self.directory, ignore_errors=True)
|
|
126
183
|
|
|
127
|
-
The bytes hash to the sha256 the platform recorded when the host exported them,
|
|
128
|
-
so they are the store's own copy, every row audited as it committed, and the
|
|
129
|
-
session opens on them without a second row audit. :meth:`verified` is the only
|
|
130
|
-
way to get one. The call log rides in the file as ``_gateway_calls``, a table the
|
|
131
|
-
store never opens: :meth:`carry_calls` writes it into a fresh export, and
|
|
132
|
-
:meth:`verified` reads it back and drops it before the store sees the file. What
|
|
133
|
-
it names is fixed once :meth:`verified` returns."""
|
|
134
184
|
|
|
135
|
-
|
|
185
|
+
_VERIFIED = object()
|
|
186
|
+
_RECOVERED = object()
|
|
187
|
+
|
|
136
188
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
189
|
+
class SessionExport:
|
|
190
|
+
"""A checkpoint the host continues a session from, opened without a second row audit.
|
|
191
|
+
|
|
192
|
+
Either an export whose bytes hash to the sha256 the platform recorded when the host
|
|
193
|
+
exported them (:meth:`verified`), so they are the store's own copy, every row
|
|
194
|
+
audited as it committed; or the committed state of a session's own store a dead
|
|
195
|
+
worker of the same version left on this host (:meth:`recovered_from`), which no
|
|
196
|
+
platform recorded: it was the live session's file, and the contract and DDL are
|
|
197
|
+
checked as for any open. The call log rides in an export as
|
|
198
|
+
``_gateway_calls``, a table the store never opens: :meth:`carry_calls` writes it
|
|
199
|
+
into a fresh export, and :meth:`verified` reads it back and drops it before the
|
|
200
|
+
store sees the file; a recovered checkpoint reads it from the call journal and
|
|
201
|
+
carries each dependency world's own checkpoint under ``worlds``. What it names is
|
|
202
|
+
fixed once it is made."""
|
|
203
|
+
|
|
204
|
+
__slots__ = ("_path", "_calls", "_worlds", "_unsent")
|
|
205
|
+
|
|
206
|
+
def __init__(
|
|
207
|
+
self,
|
|
208
|
+
path: Path,
|
|
209
|
+
calls: list[dict],
|
|
210
|
+
proof: object,
|
|
211
|
+
worlds: dict[str, SessionExport] | None = None,
|
|
212
|
+
unsent: list[dict] | None = None,
|
|
213
|
+
) -> None:
|
|
214
|
+
if proof is not _VERIFIED and proof is not _RECOVERED:
|
|
215
|
+
raise TypeError("a SessionExport comes from SessionExport.verified or SessionExport.recovered_from")
|
|
140
216
|
self._path = Path(path)
|
|
141
217
|
self._calls = tuple(calls)
|
|
218
|
+
self._worlds = dict(worlds or {})
|
|
219
|
+
self._unsent = tuple(unsent or ())
|
|
142
220
|
|
|
143
221
|
@property
|
|
144
222
|
def path(self) -> Path:
|
|
@@ -148,11 +226,23 @@ class SessionExport:
|
|
|
148
226
|
def calls(self) -> list[dict]:
|
|
149
227
|
return list(self._calls)
|
|
150
228
|
|
|
229
|
+
@property
|
|
230
|
+
def unsent(self) -> list[dict]:
|
|
231
|
+
"""The request ledger entries a dead worker had not sent; an export carries none."""
|
|
232
|
+
return list(self._unsent)
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def worlds(self) -> dict[str, SessionExport]:
|
|
236
|
+
"""Each dependency world's checkpoint, by alias; an export carries none."""
|
|
237
|
+
return dict(self._worlds)
|
|
238
|
+
|
|
151
239
|
@staticmethod
|
|
152
240
|
def required(export: object) -> SessionExport:
|
|
153
|
-
"""``export`` itself, refused unless :meth:`verified` made it."""
|
|
241
|
+
"""``export`` itself, refused unless :meth:`verified` or :meth:`recovered_from` made it."""
|
|
154
242
|
if not isinstance(export, SessionExport):
|
|
155
|
-
raise TypeError(
|
|
243
|
+
raise TypeError(
|
|
244
|
+
"a session resumes only from a SessionExport (SessionExport.verified, SessionExport.recovered_from)"
|
|
245
|
+
)
|
|
156
246
|
return export
|
|
157
247
|
|
|
158
248
|
@classmethod
|
|
@@ -161,6 +251,25 @@ class SessionExport:
|
|
|
161
251
|
calls = [json.loads(entry) for entry in engine.take_entries(export.path)]
|
|
162
252
|
return cls(export.path, calls, _VERIFIED)
|
|
163
253
|
|
|
254
|
+
@classmethod
|
|
255
|
+
def recovered_from(
|
|
256
|
+
cls, directory: Path, engine: type[WorldDatabase], live: Path, dependencies: dict[str, Path]
|
|
257
|
+
) -> SessionExport:
|
|
258
|
+
"""The committed state of the store at ``live`` and of each dependency world's
|
|
259
|
+
(a transaction their dead writer left half written undone by the engine), each
|
|
260
|
+
copied beside it, with the calls the session's journal kept."""
|
|
261
|
+
if not live.is_file() or live.is_symlink():
|
|
262
|
+
fail("host", f"the recovered session holds no store at {live.name}")
|
|
263
|
+
worlds = {alias: cls(cls._committed(store, engine), [], _RECOVERED) for alias, store in dependencies.items()}
|
|
264
|
+
calls, unsent = CallJournal.read(directory / CALLS_JOURNAL), CallJournal.read(directory / LEDGER_JOURNAL)
|
|
265
|
+
return cls(cls._committed(live, engine), calls, _RECOVERED, worlds, unsent)
|
|
266
|
+
|
|
267
|
+
@staticmethod
|
|
268
|
+
def _committed(live: Path, engine: type[WorldDatabase]) -> Path:
|
|
269
|
+
copy = live.with_name(f"recovered{Path(engine.snapshot_name).suffix}")
|
|
270
|
+
engine.recover_copy(live, copy)
|
|
271
|
+
return copy
|
|
272
|
+
|
|
164
273
|
@staticmethod
|
|
165
274
|
def carry_calls(path: Path, calls: list[dict], engine: type[WorldDatabase]) -> None:
|
|
166
275
|
engine.carry_entries(path, [json.dumps(call, sort_keys=True) for call in calls])
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""A session's way in while it changes hosts.
|
|
2
|
+
|
|
3
|
+
:class:`SessionMove` gates every request of one session that takes a turn. On the host a session
|
|
4
|
+
leaves (:mod:`host_handoff`) it goes ``serving`` → ``fenced`` (new requests wait, those inside
|
|
5
|
+
finish) → ``asking`` (quiet and settled: the platform may take its final export now) → ``moved``
|
|
6
|
+
(its requests go one hop to the new owner); back to ``serving`` when the platform keeps it here,
|
|
7
|
+
``closed`` when the platform says it is gone, ``shut`` when the host stops before it could move.
|
|
8
|
+
On the host it goes to, the platform opens it ``pending``: that copy serves nothing until the
|
|
9
|
+
platform's row names this host, which a platform verb says, or the owner lookup does for a request.
|
|
10
|
+
A fence the host lifts on its own (an ask turned away, a step that failed) puts the session back as
|
|
11
|
+
it was, a pending copy back to pending; only the platform's word makes a copy serve.
|
|
12
|
+
|
|
13
|
+
A data-plane request a move holds waits at most ``hold_s``, counted against its version's and its
|
|
14
|
+
session's waiting caps (:meth:`Admission.waiting`), then is told to come back (``503 moving`` with
|
|
15
|
+
``Retry-After``). A platform verb is never held: it is refused ``moving`` at once, and the platform
|
|
16
|
+
asks again where its row says the session is."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import http.client
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from collections.abc import Callable, Iterator
|
|
24
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
25
|
+
from typing import TYPE_CHECKING
|
|
26
|
+
|
|
27
|
+
from .host_workers import MOVING, NOT_MOVING, UNKNOWN_SESSION
|
|
28
|
+
from .validation import ContractError, fail
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from .world_host import WorldHost
|
|
32
|
+
|
|
33
|
+
SERVING = "serving"
|
|
34
|
+
PENDING = "pending"
|
|
35
|
+
FENCED = "fenced"
|
|
36
|
+
ASKING = "asking"
|
|
37
|
+
MOVED = "moved"
|
|
38
|
+
CLOSED = "closed"
|
|
39
|
+
SHUT = "shut"
|
|
40
|
+
HELD = frozenset({FENCED, ASKING})
|
|
41
|
+
|
|
42
|
+
HOLD_S = 25.0
|
|
43
|
+
MOVING_RETRY_AFTER_S = 2
|
|
44
|
+
PENDING_CHECK_S = 1.0
|
|
45
|
+
PENDING_LEASE_S = 300.0
|
|
46
|
+
|
|
47
|
+
Waiting = Callable[[], AbstractContextManager[None]]
|
|
48
|
+
OwnerNow = Callable[[], "str | None"]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def same_host(one: str | None, other: str | None) -> bool:
|
|
52
|
+
return one is not None and other is not None and one.rstrip("/") == other.rstrip("/")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def moving_refusal(session_id: str) -> ContractError:
|
|
56
|
+
return ContractError(
|
|
57
|
+
MOVING,
|
|
58
|
+
f"session {session_id} is moving to another host; retry after {MOVING_RETRY_AFTER_S} s",
|
|
59
|
+
retryAfter=MOVING_RETRY_AFTER_S,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class SessionMoved(Exception):
|
|
64
|
+
"""A request for a session that moved to ``owner``: it is sent on there, one hop."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, session_id: str, owner: str) -> None:
|
|
67
|
+
super().__init__(f"session {session_id} moved to {owner}")
|
|
68
|
+
self.session_id = session_id
|
|
69
|
+
self.owner = owner
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class SessionMove:
|
|
73
|
+
"""One session's state while it changes hosts, and the way in that state gives its requests."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, session_id: str, *, pending: bool = False, hold_s: float = HOLD_S) -> None:
|
|
76
|
+
self.session_id = session_id
|
|
77
|
+
self.hold_s = hold_s
|
|
78
|
+
self._state = PENDING if pending else SERVING
|
|
79
|
+
self._owner: str | None = None
|
|
80
|
+
self._inside = 0
|
|
81
|
+
self._asked: tuple[float, str | None] = (float("-inf"), None)
|
|
82
|
+
self._exported = False
|
|
83
|
+
self._fenced_from = SERVING
|
|
84
|
+
self._changed = threading.Condition()
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def state(self) -> str:
|
|
88
|
+
with self._changed:
|
|
89
|
+
return self._state
|
|
90
|
+
|
|
91
|
+
@contextmanager
|
|
92
|
+
def request(self, waiting: Waiting, owner_now: OwnerNow, me: str | None, hopped: bool) -> Iterator[None]:
|
|
93
|
+
"""A data-plane request's way in: a pending copy serves it once the platform names this host, a
|
|
94
|
+
moving session keeps it up to ``hold_s``, a moved one sends it on (:class:`SessionMoved`)."""
|
|
95
|
+
self._confirm(owner_now, me, hopped)
|
|
96
|
+
with self._changed:
|
|
97
|
+
self._wait_out(waiting)
|
|
98
|
+
if self._state == MOVED:
|
|
99
|
+
raise SessionMoved(self.session_id, str(self._owner))
|
|
100
|
+
self._let_in()
|
|
101
|
+
try:
|
|
102
|
+
yield
|
|
103
|
+
finally:
|
|
104
|
+
self._leave()
|
|
105
|
+
|
|
106
|
+
@contextmanager
|
|
107
|
+
def verb(self) -> Iterator[None]:
|
|
108
|
+
"""A platform verb's way in, never held: refused ``moving`` while the session moves or once it moved."""
|
|
109
|
+
with self._changed:
|
|
110
|
+
self._let_in()
|
|
111
|
+
try:
|
|
112
|
+
yield
|
|
113
|
+
finally:
|
|
114
|
+
self._leave()
|
|
115
|
+
|
|
116
|
+
def _wait_out(self, waiting: Waiting) -> None:
|
|
117
|
+
if self._state not in HELD:
|
|
118
|
+
return
|
|
119
|
+
with waiting():
|
|
120
|
+
self._changed.wait_for(lambda: self._state not in HELD, self.hold_s)
|
|
121
|
+
|
|
122
|
+
def _let_in(self) -> None:
|
|
123
|
+
if self._state == CLOSED:
|
|
124
|
+
fail(UNKNOWN_SESSION, f"no session {self.session_id} on this host")
|
|
125
|
+
if self._state != SERVING:
|
|
126
|
+
raise moving_refusal(self.session_id)
|
|
127
|
+
self._inside += 1
|
|
128
|
+
|
|
129
|
+
def _leave(self) -> None:
|
|
130
|
+
with self._changed:
|
|
131
|
+
self._inside -= 1
|
|
132
|
+
self._changed.notify_all()
|
|
133
|
+
|
|
134
|
+
def _confirm(self, owner_now: OwnerNow, me: str | None, hopped: bool) -> None:
|
|
135
|
+
if self.state != PENDING:
|
|
136
|
+
return
|
|
137
|
+
owner = self._pending_owner(owner_now, hopped)
|
|
138
|
+
if same_host(owner, me):
|
|
139
|
+
self.commit()
|
|
140
|
+
return
|
|
141
|
+
if owner is None:
|
|
142
|
+
fail(UNKNOWN_SESSION, f"no session {self.session_id} on this host")
|
|
143
|
+
raise SessionMoved(self.session_id, owner)
|
|
144
|
+
|
|
145
|
+
def _pending_owner(self, owner_now: OwnerNow, hopped: bool) -> str | None:
|
|
146
|
+
"""Who the platform says owns the session, asked at most once every ``PENDING_CHECK_S`` unless
|
|
147
|
+
another host sent the request here, which it does only once the session moved here."""
|
|
148
|
+
with self._changed:
|
|
149
|
+
asked_at, owner = self._asked
|
|
150
|
+
if not hopped and time.monotonic() - asked_at < PENDING_CHECK_S:
|
|
151
|
+
return owner
|
|
152
|
+
try:
|
|
153
|
+
owner = owner_now()
|
|
154
|
+
except (OSError, ValueError, http.client.HTTPException) as error:
|
|
155
|
+
raise moving_refusal(self.session_id) from error
|
|
156
|
+
with self._changed:
|
|
157
|
+
self._asked = (time.monotonic(), owner)
|
|
158
|
+
return owner
|
|
159
|
+
|
|
160
|
+
def commit(self) -> None:
|
|
161
|
+
"""A pending copy the platform's row names: it serves from now on, or once a fence is lifted."""
|
|
162
|
+
with self._changed:
|
|
163
|
+
self._fenced_from = SERVING
|
|
164
|
+
self._become(SERVING, {PENDING})
|
|
165
|
+
|
|
166
|
+
def fence(self, within_s: float) -> bool:
|
|
167
|
+
"""Hold every new request, remembering what the session was; True once those already inside have
|
|
168
|
+
left, within ``within_s``."""
|
|
169
|
+
with self._changed:
|
|
170
|
+
if self._state in (SERVING, PENDING):
|
|
171
|
+
self._fenced_from, self._state = self._state, FENCED
|
|
172
|
+
return self._state == FENCED and self._changed.wait_for(lambda: self._inside == 0, max(0.0, within_s))
|
|
173
|
+
|
|
174
|
+
def ask(self) -> None:
|
|
175
|
+
self._become(ASKING, {FENCED})
|
|
176
|
+
|
|
177
|
+
def stay(self) -> None:
|
|
178
|
+
"""The platform keeps the session on this host, whose copy its row names: it serves from now on,
|
|
179
|
+
and the export a move took is moot."""
|
|
180
|
+
with self._changed:
|
|
181
|
+
self._exported = False
|
|
182
|
+
self._fenced_from = SERVING
|
|
183
|
+
self._become(SERVING, HELD)
|
|
184
|
+
|
|
185
|
+
def unfence(self) -> bool:
|
|
186
|
+
"""Lift a fence on this host's own word (an ask turned away, a step that failed): the session is
|
|
187
|
+
again what it was, serving or a copy pending a move here. Refused while the platform holds its
|
|
188
|
+
final export, which a move may still publish elsewhere."""
|
|
189
|
+
with self._changed:
|
|
190
|
+
if self._exported:
|
|
191
|
+
return False
|
|
192
|
+
self._become(self._fenced_from, HELD)
|
|
193
|
+
return True
|
|
194
|
+
|
|
195
|
+
def closing(self) -> None:
|
|
196
|
+
"""The platform closes the session here: refused ``moving`` while a move holds its final export,
|
|
197
|
+
which another host may yet serve; from then on no move exports it."""
|
|
198
|
+
with self._changed:
|
|
199
|
+
if self._exported:
|
|
200
|
+
raise moving_refusal(self.session_id)
|
|
201
|
+
self._become(CLOSED, {SERVING, PENDING, FENCED, ASKING, SHUT})
|
|
202
|
+
|
|
203
|
+
def moved(self, owner: str) -> None:
|
|
204
|
+
with self._changed:
|
|
205
|
+
self._owner = owner
|
|
206
|
+
self._become(MOVED, {SERVING, PENDING, FENCED, ASKING, SHUT})
|
|
207
|
+
|
|
208
|
+
def closed(self) -> None:
|
|
209
|
+
self._become(CLOSED, {SERVING, PENDING, FENCED, ASKING, SHUT})
|
|
210
|
+
|
|
211
|
+
def shut(self) -> None:
|
|
212
|
+
"""The host stops before this session could move: its requests are told to come back from now on."""
|
|
213
|
+
self._become(SHUT, {SERVING, PENDING, FENCED, ASKING})
|
|
214
|
+
|
|
215
|
+
def _become(self, state: str, after: set[str] | frozenset[str]) -> None:
|
|
216
|
+
with self._changed:
|
|
217
|
+
self._state = state if self._state in after else self._state
|
|
218
|
+
self._changed.notify_all()
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def pending(self) -> bool:
|
|
222
|
+
return self.state == PENDING
|
|
223
|
+
|
|
224
|
+
@property
|
|
225
|
+
def leaving(self) -> bool:
|
|
226
|
+
return self.state in (FENCED, ASKING, MOVED, SHUT)
|
|
227
|
+
|
|
228
|
+
def check_export(self, handoff: bool) -> None:
|
|
229
|
+
"""A handoff export is answered only once the session is asking to move, and marks it taken; no
|
|
230
|
+
other export while it moves."""
|
|
231
|
+
with self._changed:
|
|
232
|
+
if handoff and self._state != ASKING:
|
|
233
|
+
fail(NOT_MOVING, f"this host is not handing session {self.session_id} off")
|
|
234
|
+
if not handoff and self._state in HELD:
|
|
235
|
+
raise moving_refusal(self.session_id)
|
|
236
|
+
self._exported = self._exported or handoff
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def platform_close(host: WorldHost, session_id: str) -> dict:
|
|
240
|
+
"""The platform's close of a session (``DELETE``, ``POST close``), which goes where its row points:
|
|
241
|
+
while a move holds the final export here, the host says ``moving`` and the platform asks again."""
|
|
242
|
+
session = host.held(session_id)
|
|
243
|
+
if session is not None:
|
|
244
|
+
session.move.closing()
|
|
245
|
+
return host.close(session_id)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class Forwarding:
|
|
249
|
+
"""Where each session this host handed off went, so a request for it that still lands here
|
|
250
|
+
(a stale route, an owner another replica remembers) is sent on there, or, from the platform,
|
|
251
|
+
told the session is moving rather than lost."""
|
|
252
|
+
|
|
253
|
+
def __init__(self, self_url: str | None = None) -> None:
|
|
254
|
+
self.self_url = self_url
|
|
255
|
+
self._owners: dict[str, str] = {}
|
|
256
|
+
self._lock = threading.Lock()
|
|
257
|
+
|
|
258
|
+
def record(self, session_id: str, owner: str) -> None:
|
|
259
|
+
if same_host(owner, self.self_url):
|
|
260
|
+
raise ValueError(f"session {session_id} cannot move to the host it leaves")
|
|
261
|
+
with self._lock:
|
|
262
|
+
self._owners[session_id] = owner
|
|
263
|
+
|
|
264
|
+
def owner(self, session_id: str) -> str | None:
|
|
265
|
+
with self._lock:
|
|
266
|
+
return self._owners.get(session_id)
|
|
267
|
+
|
|
268
|
+
def refuse(self, session_id: str) -> None:
|
|
269
|
+
if self.owner(session_id) is not None:
|
|
270
|
+
raise moving_refusal(session_id)
|
|
271
|
+
|
|
272
|
+
def instead_of(self, error: ContractError, session_id: str | None) -> ContractError:
|
|
273
|
+
"""A session this host handed off answers ``moving``, never the ``unknown_session`` the platform reads as a loss."""
|
|
274
|
+
handed_off = session_id is not None and self.owner(session_id) is not None
|
|
275
|
+
return moving_refusal(str(session_id)) if handed_off and error.detail.get("code") == UNKNOWN_SESSION else error
|
|
@@ -93,7 +93,11 @@ ship `connector.toml` in the template directory. `init` copies it to the world r
|
|
|
93
93
|
The format is documented in `SKILL.md` under "Custom connections" and "Serving the vendor's API":
|
|
94
94
|
|
|
95
95
|
- `[connector]` slug and `base_url`; `[auth]` scheme (`header|bearer|basic|query|oauth2|none`),
|
|
96
|
-
`name`, and `secret_env` naming the environment variable, never a value
|
|
96
|
+
`name`, and `secret_env` naming the environment variable, never a value; `prefixes` when the
|
|
97
|
+
vendor takes other `Authorization` words than `Bearer`, `credential = "password"` when its Basic
|
|
98
|
+
key is the password; `[auth.login]` when the vendor trades its key for a token,
|
|
99
|
+
`[auth.downloads]` when it hands out links to files; `[api] unauthorized_status` and
|
|
100
|
+
`unauthorized` as the vendor refuses a bad key.
|
|
97
101
|
- `[capture]` pacing (`requests_per_minute` low; providers ask us to be sparing), cursor
|
|
98
102
|
pagination when the vendor has it (`cursor_field` in the body or `cursor_header`);
|
|
99
103
|
`[redact]` `hash`/`drop` for every PII field the vendor documents returning, `round` for
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: benchmarks-getting-started
|
|
3
|
-
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
3
|
+
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state and the requests it received) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Benchmarks: getting started
|
|
@@ -355,13 +355,12 @@ gateway worlds schema task add ./crm-bench close_ticket --prompt "Close ticket T
|
|
|
355
355
|
gateway worlds schema task add ./crm-bench escalate --prompt "..." --verifier state_present --world slack=state_present
|
|
356
356
|
```
|
|
357
357
|
|
|
358
|
-
- `--prompt` is what the agent is told. `--verifier <name>` names `verifiers/<name>.sql`; it must
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
`
|
|
362
|
-
- A verifier
|
|
363
|
-
|
|
364
|
-
"crm_tickets" WHERE id = 'T-7' AND status = 'closed';`
|
|
358
|
+
- `--prompt` is what the agent is told. `--verifier <name>` names `verifiers/<name>.sql`; it must exist — `schema verifier add <dir> <name> --entity <entity> [--where "<sql condition>"]` scaffolds
|
|
359
|
+
it (without `--where`, the `state_present` placeholder). Each verifier becomes a key in `rewards`; `reward` is their mean.
|
|
360
|
+
- A verifier is one SQL statement returning one number in 0..1 over the world's tables (entity names from `schema/world.json`):
|
|
361
|
+
`SELECT CASE WHEN COUNT(*) > 0 THEN 1.0 ELSE 0.0 END FROM "crm_tickets" WHERE id = 'T-7' AND status = 'closed';`
|
|
362
|
+
- A verifier can also read every request the session's world received, in `gateway_requests` (and its summary `gateway_journal`):
|
|
363
|
+
`SELECT CASE WHEN COUNT(*) = 0 THEN 1.0 ELSE 0.0 END FROM gateway_requests WHERE method = 'DELETE';`. Only a World Host session records them (`dry-run` and `validate` read them as not recorded, and such a verifier scores 0 there whatever it accepts); on a journal that may be missing requests it scores 0 with the reason unless `gateway-env.toml` has `[verifiers.<name>] accepts_incomplete_journal = true`, which suits a check a missing request can only fail (`COUNT(*) > 0`), never `= 0`. A linked world's verifier gets no journal, so one that reads it scores 0. Columns, fail-closed rules, declarative and `verify.py` checks over the same requests: `worlds-getting-started` `references/journal-checks.md`.
|
|
365
364
|
- `--world <alias>=<verifier>` grades a linked world's state too (`schema link` first).
|
|
366
365
|
- Stored task specs (`instruction`, `seed`, `grader` of kind `assertions` | `python` | `rubric`):
|
|
367
366
|
`gateway worlds task create task.json --world <slug>`; `worlds-getting-started` §6.
|
|
@@ -496,5 +495,5 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
496
495
|
|
|
497
496
|
- `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
|
|
498
497
|
- `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
|
|
499
|
-
- Consumer pages: `
|
|
498
|
+
- Consumer pages (docs.surfacearea.ai): `cli/benchmarks`, `worlds/benchmarks`, `evaluation/benchmarks-harbor`.
|
|
500
499
|
- `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
|
package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md
CHANGED
|
@@ -68,9 +68,9 @@ real row; captured exports from that tenant become the goldens.
|
|
|
68
68
|
| Slug | State | Notes |
|
|
69
69
|
| --- | --- | --- |
|
|
70
70
|
| slack | package 2.1.1 / domain 2.0.0 / mapping 1, spec-derived fields (wave 1) | base; 34 normalized entities / 72 `$defs` / 27 tools; `connector.toml` serves 56 Web API operations from `api/handlers.py` (bearer `SLACK_BOT_TOKEN`, Slack `ok`/`error` envelopes); `scenarios/default` seeds 34 entities / 84 synthetic rows; `vendor-openapi.json` (trimmed, response schemas relocated to OpenAPI 3 `content`) — `connector conform` 51/54 documented operations pass, 3 fail on channel-keyed `file.shares.public` that the 2020 spec stubs empty, `bookmarks.*` undocumented there. `subset`: no live round trip. Empty init or populated synthetic/selected capture → immutable hosted world; [on-demand fixtures](../../../samples/schema-worlds/README.md#slack-fixtures) |
|
|
71
|
-
| github | spec-derived, wave 2 (domain 4.0.0) | base; pinned GitHub REST OpenAPI (`adb01e9`, 2026-09-15) plus the GraphQL review-state declaration. 45 entities / 5,365 recursive fields / 4 tools / 203 served operations (`connector.toml` + `api/handlers.py`; `scenarios/default` seeds 111 rows across every entity; `vendor-openapi.json` trimmed to the served paths, 203/203 conform). The fullest vendor read form is each entity's root (full-repository, public-user, organization-full, team-full, ...) with the narrower forms' required lists intersected where scope.toml lists several read_forms; route containment and nested-scalar projections are direct fields so keys and relationships are enforceable; many-to-many links are membership entities. Issue state/state-reason come from declared REST sibling value blocks; review state from `PullRequestReviewState`; user `type` stays open. 1,026 documented routes unserved, pagination not emulated; `subset` until a test-org round trip and synthetic goldens exist. |
|
|
71
|
+
| github | spec-derived, wave 2 (domain 4.0.0) | base; pinned GitHub REST OpenAPI (`adb01e9`, 2026-09-15) plus the GraphQL review-state declaration. 45 entities / 5,365 recursive fields / 4 tools / 203 served operations (`connector.toml` + `api/handlers.py`; `scenarios/default` seeds 111 rows across every entity; `vendor-openapi.json` trimmed to the served paths, 203/203 conform). The fullest vendor read form is each entity's root (full-repository, public-user, organization-full, team-full, ...) with the narrower forms' required lists intersected where scope.toml lists several read_forms; route containment and nested-scalar projections are direct fields so keys and relationships are enforceable; many-to-many links are membership entities. Issue state/state-reason come from declared REST sibling value blocks; review state from `PullRequestReviewState`; user `type` stays open. 1,026 documented routes unserved, pagination not emulated; `subset` until a test-org round trip and synthetic goldens exist. Sends GitHub's webhooks (`[webhooks]`, `github` signing): `ping`, `issues`, `issue_comment`, `pull_request`, `push`; JSON only, repository hooks only. |
|
|
72
72
|
| salesforce | docs-derived, wave 2 (full-parity expansion) | tenant-shaped skeleton from 99 pinned Winter '27 (API 68.0, doc 264.0) Object Reference pages plus 32 REST API Developer Guide resource pages: 99 entities / 2,519 top-level fields (2,695 recursive) / 215 restrict relationships / 102 closed enums / 14 unique constraints / 4 tools; `connector.toml` serves 332 operations (Resources by Version, Describe Global, sObject Basic Information, sObject Describe, `/query` SOQL subset, `/search` SOSL subset, and per-object retrieve/create/update/delete exactly where each page's Supported Calls lists them) with `api/handlers.py` for the computed routes; `scenarios/default` seeds 2 synthetic rows in every entity (198 rows). **Describe-on-bind is required** for the org's actual fields, relationships, picklists, custom schema, record types and permissions; the served `/describe` answers from the skeleton and says so. Only Restricted picklists with an explicit vendor value block become enums; org-controlled picklists (StageName, Lead.Status, Case Status/Priority/Origin, ...) remain open strings. Single-target `Refers To` fields become relationships; polymorphic (`WhoId`, `WhatId`, Group-or-User owners: 39) and references to unmodelled objects (183) remain ids. Not served, reported: upsert by external id, deleted/updated, queryAll, composite, limits, Bulk/Tooling/Metadata/Streaming, blob bodies. Conformance not applicable: Salesforce publishes only an org-generated OAS3 beta. `subset`: no org describe, Developer Edition/scratch-org round trip or goldens. |
|
|
73
|
-
| linear | SDL-derived, wave 2 | base; transcribed from the pinned linear/linear `packages/sdk/src/schema.graphql` (commit `23f11eb4`). 32 entities / 600 fields / 114 restrict relationships / 16 enums / 4 tools; 150 REST-shaped `connector.toml` operations (55 read, 24 create, 28 update, 23 delete, 20 handler) with a synthetic `default` scenario for every entity. Vendor camelCase is kept; GraphQL non-null is required, nullable is required-but-null. `Project.state` stays despite `@deprecated` because the brief requires it; `Issue.priority` is integer 0..4 from the description. `linear_roadmaps` has no routes because every roadmap root field is `@deprecated`. OpenAPI conformance is not applicable (GraphQL only, no vendor OpenAPI). `subset`: 90 Node types excluded, 28 generated `<field>Id` names unconfirmed, no live introspection or workspace round trip. |
|
|
73
|
+
| linear | SDL-derived, wave 2 | base; transcribed from the pinned linear/linear `packages/sdk/src/schema.graphql` (commit `23f11eb4`). 32 entities / 600 fields / 114 restrict relationships / 16 enums / 4 tools; 150 REST-shaped `connector.toml` operations (55 read, 24 create, 28 update, 23 delete, 20 handler) with a synthetic `default` scenario for every entity. Vendor camelCase is kept; GraphQL non-null is required, nullable is required-but-null. `Project.state` stays despite `@deprecated` because the brief requires it; `Issue.priority` is integer 0..4 from the description. `linear_roadmaps` has no routes because every roadmap root field is `@deprecated`. OpenAPI conformance is not applicable (GraphQL only, no vendor OpenAPI). `subset`: 90 Node types excluded, 28 generated `<field>Id` names unconfirmed, no live introspection or workspace round trip. Sends Linear's webhooks (`[webhooks]`, `linear` signing) for `Issue` and `Comment` create/update/remove, filtered by `resourceTypes` and team. |
|
|
74
74
|
| jira | spec-derived, schema-worlds pass | mixed; `$defs` transcribed from the pinned Jira Cloud platform REST v3 OpenAPI (1001.0.0-SNAPSHOT, 2,473,385 bytes) and the Jira Software Cloud OpenAPI (1001.0.0) for boards/sprints. 21 entities / 65 `$defs` (477 properties) / 32 relationships / 5 tools / 78 `connector.toml` operations (78/78 conform against the trimmed `vendor-openapi.json`). Issues are keyed by immutable `id` with `key` unique; transitions by (`issueId`, `id`); sprint membership is a link entity. `IssueBean.fields` is `{}` in the vendor document, so an authored `IssueFields` `$def` types 38 system-field members and stays open for `customfield_*` overlay values. Enums only where the vendor declares them (`projectTypeKey`, `accountType`, `style`, `assigneeType`, sprint `state` from the `getAllSprints` filter); `StatusCategory.key` stays open. `scenarios/default` seeds every entity synthetically. `subset`: no Jira Cloud round trip, no goldens, Rovo MCP tool parity unverified. |
|
|
75
75
|
| microsoft-teams | authored base, wave 3 (served) | [Usage, artifacts, scope, and pinned evidence](../../../documentation/features/microsoft-teams-schema.md). 19 entities / 81 vendor `$defs` / 0 tools; teams, channels (incoming, shared-with), chats, chat views, team/channel/chat members, channel messages, replies, chat messages, hosted contents, tags, tag members, pinned messages, async operations. `connector.toml`: 99 Graph v1.0 operations (`auth.scheme = "none"`), all `action = "handler"` because Graph answers carry `@odata.*` annotations and hide the capture-context key column; documented statuses 200/201/202/204/207. `scenarios/default`: 65 synthetic rows. `vendor-openapi.json`: pinned Graph OpenAPI trimmed to the 68 served path items (302 KB) with `2XX`/`204` success keys rewritten to the documented concrete statuses (listed in the sidecar); `connector conform` 98/99, the miss being `DELETE .../incomingChannels/{id}/$ref` which the docs describe but the OpenAPI omits. Not served: user-scoped aliases, `getAllMessages`/delta exports, `$value` binaries, apps/tabs/photo/schedule/permissionGrants. `subset`: no sandbox round trip or goldens. |
|
|
76
76
|
| google-drive | discovery-derived, wave 2 | base; derived from the pinned Drive v3 Discovery document (revision 20260904, sha256 `6d38b6c8e1d1a45fa1b30744523dc6c400ad612997f8e67d112271b573e4948e`), no vendor OpenAPI exists. 10 entities / 34 `$defs` / 410 recursive fields / 4 tools; `drive_folders` is gone (folders are files by `mimeType`). All ten documented resources are entities (Drive, File, Permission, Revision, Comment, Reply, Label, App, AccessProposal, Approval); 49 of 64 documented methods are served from `connector.toml` (changes, channels, about, export/download, CSE tokens, operations and the deprecated `teamdrives.*` are not). Child rows carry route-context `fileId`/`commentId` keys; `parents` stays the vendor array and is not a relationship. Enums are the vendor's declared arrays plus the value lists its descriptions state, recorded separately in `provenance.json`. `vendor-openapi.json` is generated from the Discovery document and trimmed to the served operations; `connector conform` 49/49 against it after seeding `scenarios/default`. `subset`: prose-enum completeness and list-envelope constants unverified, no test-domain round trip or goldens. |
|
|
@@ -80,7 +80,7 @@ real row; captured exports from that tenant become the goldens.
|
|
|
80
80
|
| custom | hand-authored, worked example | the blank starting point for any arbitrary connector, now a worked example of the authoring contract with no vendor behind it: 1 entity (`records`, the only name the scaffold test allows) / 15 fields / 3 tools, showing a primary key, a sparse unique key (`slug`), a self-relationship (`parent_id -> records`, restrict), an enum in `$defs` (`status`), optional (`owner_email`) vs optional-nullable (`parent_id`, `priority`, `due_at`) vs required-nullable (`source.external_id`), a closed nested object and an open bag. Only the original five fields (`id`, `kind`, `name`, `attributes`, `updated_at`) are required and `kind` stays an open string: the SDK test suite uses this template as its blank fixture and inserts rows of that shape, so the example grows by optional fields only. No `connector.toml` ships: the redaction tests need a policy-free world and the same tests create `connector.toml`, `api/` and `scenarios/` themselves and read back an exact template file set, so the template also ships no `api/handlers.py`, no `scenarios/default` seed and no `vendor-openapi.json` (conformance not applicable). Everything is synthetic; `schema init --connector custom` renames the world and drops the tools unless `--with-example-tool`. |
|
|
81
81
|
| workday | hand-authored, wave 3 + REST expansion | tenant-shaped skeleton from pinned public WWS v46.2 WSDL/XSD files (Human Resources, Staffing, Absence Management, Compensation) plus five pinned public Workday REST Swagger 2.0 documents (common v1, staffing v7, absenceManagement v5, compensation v3, person v4, dated 2026-09-12): 27 entities (7 `workday_*` WWS + 20 `workday_rest_*`) / 729 recursive fields / 14 relationships / 3 tools. `connector.toml` serves 62 of the 332 documented REST operations (OAuth2, offset/limit `data[]` envelope, PII redaction, six handler-backed workflow routes); unmodelled routes are listed in `provenance.json`. `scenarios/default` seeds synthetic rows for every entity; `vendor-openapi.json` (OAS 3.0.3 converted and trimmed from the Swagger sources) conforms 62/62. **Describe-on-bind is required** for an actual tenant (tenant WSDL, REST host and OAuth client, custom objects/fields, RaaS). WWS and REST identifier systems stay separate entities until a tenant overlay maps them. Workday-namespaced `wd:enumeration` blocks are used only where the selected reference-ID types declare them; tenant business values remain open. `xsd:date` maps to `format: date`; `xsd:time` remains unformatted because the source permits a missing timezone. `subset`: no tenant round trip, no live capture, timeTracking v6 unreachable. |
|
|
82
82
|
| kandji | hand-authored, wave 3 (expanded 2026-09-15) | base; transcribed from the pinned vendor Postman collection rendered by api-docs.iru.com (135 requests), with the community OpenAPI (`MScottBlake/kandji-openapi@46fa742`) used only as a cross-check. 21 entities / 234 recursive fields / 25 `$defs` / 3 tools; `connector.toml` serves 48 operations (49 of the 135 documented requests: devices, apps, library-item statuses, parameters, commands, activity, notes CRUD, blueprints CRUD + templates + library-item assignment, library-item status/activity, users, tags CRUD, custom apps/profiles/scripts, in-house apps, self-service categories) with 21 handler routes; `scenarios/default` seeds 57 synthetic rows. Device `user` stays as the vendor's nested object-or-empty-string shape; `/user/id` is enforced when assigned and the empty-string sentinel is treated as absent. Nine vendor-declared value blocks (blueprint type, library-item status, activity type, parameter status, integer command status codes, custom-app install type/enforcement, script frequency); platform values are example-derived and completeness remains unverified. Apple-family app shape only; Windows/Android forms need a polymorphic-key model. Trimmed community OpenAPI shipped as `vendor-openapi.json` (18 paths / 26 ops): `connector conform` 16/26 pass; the 10 failures are the community document's inferred types (booleans as integer, arrays as object, nullable `next` as string, blueprint DELETE 200) contradicted by the vendor examples. Not served: device details/Prism, actions, secrets, ADE, threats, vulnerabilities, audit, blueprint routing, upload handshakes. `subset`: no vendor OpenAPI, no account-team sandbox, tenant round trip, or goldens. |
|
|
83
|
-
| jamf | docs-derived, wave 4 | mixed; generated from the OpenAPI 3.0.1 fragments embedded in every pinned developer.jamf.com reference page (Jamf Pro API 11.32.0: 825 pages / 823 operations; Classic API 11.32.0: 689 pages / 613 operations; all fetched 2026-09-15 and hashed in provenance.json) because no per-instance `/api/schema` was available. 66 entities / 166 `$defs` / 2099 recursive fields / 4 tools. Pro resources keep string ids, Classic resources integer ids (root `id` = `general.id`); static and smart groups share one entity per platform; computer and mobile-device group membership are join entities with both foreign keys, prestages and DEP devices point at their device enrollment (7 relationships, all `restrict`); group-name uniqueness is asserted because the create routes document `422 Duplicate group name`, nothing else is. connector.toml serves 302 documented routes (178 Pro under `/api`, 124 Classic under `/JSSResource`; 110 handlers for paged lists, membership and group writes, 90 reads, 25 updates, 77 deletes); the 1134 documented-but-unserved operations are listed per path family in scope.toml `[unserved]`. `vendor-openapi.json` is the two aggregates trimmed to the served routes and their schemas (431 KB; untrimmed hashes in provenance.json) and `connector conform` passes 302/302 after seeding scenarios/default (66 seed files, 198 synthetic rows). Classic list responses follow the pinned array-of-wrapper schema, not the live envelope; Classic XML writes are not
|
|
83
|
+
| jamf | docs-derived, wave 4 | mixed; generated from the OpenAPI 3.0.1 fragments embedded in every pinned developer.jamf.com reference page (Jamf Pro API 11.32.0: 825 pages / 823 operations; Classic API 11.32.0: 689 pages / 613 operations; all fetched 2026-09-15 and hashed in provenance.json) because no per-instance `/api/schema` was available. 66 entities / 166 `$defs` / 2099 recursive fields / 4 tools. Pro resources keep string ids, Classic resources integer ids (root `id` = `general.id`); static and smart groups share one entity per platform; computer and mobile-device group membership are join entities with both foreign keys, prestages and DEP devices point at their device enrollment (7 relationships, all `restrict`); group-name uniqueness is asserted because the create routes document `422 Duplicate group name`, nothing else is. connector.toml serves 302 documented routes (178 Pro under `/api`, 124 Classic under `/JSSResource`; 110 handlers for paged lists, membership and group writes, 90 reads, 25 updates, 77 deletes); the 1134 documented-but-unserved operations are listed per path family in scope.toml `[unserved]`. `vendor-openapi.json` is the two aggregates trimmed to the served routes and their schemas (431 KB; untrimmed hashes in provenance.json) and `connector conform` passes 302/302 after seeding scenarios/default (66 seed files, 198 synthetic rows). Classic list responses follow the pinned array-of-wrapper schema, not the live envelope; Classic XML writes are served only for webhooks (create, update). Sends Jamf's `ComputerAdded` webhook (`[webhooks]`, unsigned) to JSON subscriptions; XML subscriptions and Basic auth are not carried. `subset` until a per-instance schema, a sandbox round trip and goldens exist. |
|
|
84
84
|
| apple-business-manager | docs-derived, wave 3 (full-parity pass) | base; generated from the complete pinned Apple Business API 2.4 DocC JSON corpus (329 pages, per-page sha256 in provenance.json). 24 entities / 281 recursive fields / 28 closed enums / 2 read tools: every documented resource type (OrgDevice, AppleCareCoverage, MdmDevice, MdmDeviceDetail, MdmServer, OrgDeviceActivity, User, UserGroup, OrganizationalUnit, Blueprint, Configuration, App, Package, AuditEvent with all eventData payloads) plus one linkage entity per documented relationship. Vendor `attributes` nesting is preserved; nested `enrolledUserId` and user-group `ouId` relationships are enforced, `roleOuList[].ouId` is not (array traversal); `productFamily` stays open because the shared hr/apple test fixture inserts `MAC`/`IPAD` against Apple's documented `Mac`/`iPad`. `connector.toml` declares all 56 documented routes (OAuth2 client credentials, cursor paging) answered by `api/handlers.py` in Apple's JSON:API envelope with `fields[type]`, `include`, `filter[...]`; `scenarios/default` seeds 69 synthetic rows across every entity. No OpenAPI exists, so `connector conform` is not applicable; the six `DELETE .../relationships/*` routes answer 400 because the world API drops DELETE bodies. `subset`: no sandbox, no live evidence, no goldens. |
|
|
85
85
|
| netsuite | docs-derived, wave 3 (expanded) | tenant-shaped skeleton from the pinned 2026.1 REST API Browser (28 chunks, sha256 in provenance.json; 2023.2 archive as cross-check) plus Oracle record-service, filtering, paging, upsert, error and OAuth docs: 22 standard records as 23 entities / 325 `$defs` / 11,385 recursive fields / 171 relationships / 3 tools / 144 `connector.toml` operations (list, create, get, PATCH, upsert `eid:`, delete per record and 12 modelled-to-modelled `!transform` routes; nsError envelope, `q` filter grammar, `limit`/`offset` paging, bearer required). **Describe-on-bind is required** for role/account-specific records, custom fields, lists, segments, enums, and nullability. Vendor nesting is preserved; `salesOrder-itemElement` is also projected as `netsuite_sales_order_lines` (`order_id`, `line`). Single-target references become restrict relationships; five `one of` references the brief requires (`entity`/`customer` on sales orders, invoices, credit memos, payments; line `item`) are narrowed to customers/items and listed in scope.toml; the other 237 and all collection-item paths stay unenforced. `externalId` uniqueness follows the `eid:` address; the brief's business numbers (`entityId`, `itemId`, `tranId`, `acctNumber`, subsidiary `name`) are declared as the world's contract, not as vendor-schema facts. Default scenario: 97 synthetic rows across all 23 entities. No `vendor-openapi.json`: the Browser is HTML and the OpenAPI 3.0 metadata is per account, so `conform` is not applicable. `suiteql` is not shipped. `subset`: no account metadata overlay, round trip, or goldens. |
|
|
86
86
|
| microsoft-teams | authored schema-only base | [Usage, artifacts, scope, and pinned evidence](../../../documentation/features/microsoft-teams-schema.md); no tools or live integration |
|