@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -79,9 +79,9 @@ from typing import Any
|
|
|
79
79
|
|
|
80
80
|
from .._toml import tomllib
|
|
81
81
|
from ..world_session_contract import ADMIN_TOKEN_ENV
|
|
82
|
-
from .
|
|
83
|
-
from .conform import PIN, _credentials
|
|
82
|
+
from .conform import PIN
|
|
84
83
|
from .connector import load_spec
|
|
84
|
+
from .door import Door
|
|
85
85
|
from .local import LocalSession, open_world
|
|
86
86
|
from .validation import fail
|
|
87
87
|
from .workers import INHERITED_ENV
|
|
@@ -108,6 +108,20 @@ class CaseResult:
|
|
|
108
108
|
problems: list[str] = field(default_factory=list)
|
|
109
109
|
|
|
110
110
|
|
|
111
|
+
@dataclass
|
|
112
|
+
class PythonCase:
|
|
113
|
+
name: str
|
|
114
|
+
file: str
|
|
115
|
+
group: str
|
|
116
|
+
status: str
|
|
117
|
+
problems: list[str] = field(default_factory=list)
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def of(cls, case, root: Path) -> "PythonCase":
|
|
121
|
+
file = _case_file(case, root)
|
|
122
|
+
return cls(name=_readable(case.get("name") or ""), file=file, group=file, status=_case_status(case), problems=_case_problems(case))
|
|
123
|
+
|
|
124
|
+
|
|
111
125
|
@dataclass
|
|
112
126
|
class PythonResult:
|
|
113
127
|
status: str
|
|
@@ -119,6 +133,8 @@ class PythonResult:
|
|
|
119
133
|
reason: str | None = None
|
|
120
134
|
output: str = ""
|
|
121
135
|
interpreter: str | None = None
|
|
136
|
+
cases: list[PythonCase] = field(default_factory=list)
|
|
137
|
+
cases_omitted: int = 0
|
|
122
138
|
|
|
123
139
|
@property
|
|
124
140
|
def ran_nothing(self) -> bool:
|
|
@@ -138,6 +154,7 @@ class Report:
|
|
|
138
154
|
failed = sum(1 for c in self.cases if c.status != "passed")
|
|
139
155
|
if self.python.status == "failed":
|
|
140
156
|
failed += max(1, self.python.failures + self.python.errors)
|
|
157
|
+
passed += max(0, self.python.tests - self.python.failures - self.python.errors - self.python.skipped)
|
|
141
158
|
if self.python.status == "passed":
|
|
142
159
|
passed += self.python.tests - self.python.skipped
|
|
143
160
|
return {
|
|
@@ -352,12 +369,12 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
|
|
|
352
369
|
)
|
|
353
370
|
spec = load_spec(root) if session.routes else None
|
|
354
371
|
prefix = urllib.parse.urlsplit(spec.base_url).path.rstrip("/") if spec else ""
|
|
355
|
-
|
|
372
|
+
door = session.app.door if session.app is not None else None
|
|
356
373
|
for file, case in cases:
|
|
357
374
|
if "tool" in case:
|
|
358
375
|
results.append(_run_tool_case(session, file, case))
|
|
359
376
|
else:
|
|
360
|
-
results.append(_run_case(base, prefix,
|
|
377
|
+
results.append(_run_case(base, prefix, door, file, case))
|
|
361
378
|
python = _run_python(root, python_files, session)
|
|
362
379
|
finally:
|
|
363
380
|
session.close()
|
|
@@ -395,7 +412,7 @@ def _run_tool_case(session: LocalSession, file: str, case: dict) -> CaseResult:
|
|
|
395
412
|
return result
|
|
396
413
|
|
|
397
414
|
|
|
398
|
-
def _run_case(base: str, prefix: str,
|
|
415
|
+
def _run_case(base: str, prefix: str, door: Door | None, file: str, case: dict) -> CaseResult:
|
|
399
416
|
request = case["request"]
|
|
400
417
|
expect = case["expect"]
|
|
401
418
|
name = str(case.get("name") or f"{request.get('method', 'GET')} {request.get('path', '/')}")
|
|
@@ -413,13 +430,11 @@ def _run_case(base: str, prefix: str, spec, credentials: dict[str, str], file: s
|
|
|
413
430
|
if prefix and not path.startswith(prefix + "/") and path != prefix and not path.startswith("/__"):
|
|
414
431
|
# the world's own /__ routes (session, ui, the ui's api_path) are never under the vendor's prefix
|
|
415
432
|
path = prefix + (path if path.startswith("/") else "/" + path)
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
433
|
+
presents = door is not None and request.get("auth", True)
|
|
434
|
+
credential_headers, credential_query = door.present() if presents else ({}, {})
|
|
435
|
+
query = {**dict(request.get("query") or {}), **credential_query}
|
|
419
436
|
url = base + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
420
|
-
headers = {k: str(v) for k, v in (request.get("headers") or {}).items()}
|
|
421
|
-
if request.get("auth", True):
|
|
422
|
-
headers = {**credentials, **headers}
|
|
437
|
+
headers = {**credential_headers, **{k: str(v) for k, v in (request.get("headers") or {}).items()}}
|
|
423
438
|
body = request.get("body")
|
|
424
439
|
data = None
|
|
425
440
|
if body is not None:
|
|
@@ -516,6 +531,7 @@ def _run_python(root: Path, files: list[Path], session: LocalSession) -> PythonR
|
|
|
516
531
|
reason = "pytest wrote no junit report"
|
|
517
532
|
return PythonResult(status="failed", files=names, reason=reason, output=output, interpreter=interpreter)
|
|
518
533
|
tests, failures, errors, skipped = counts
|
|
534
|
+
cases, omitted = _bounded(_junit_cases(junit, root))
|
|
519
535
|
if proc.returncode == PYTEST_NO_TESTS_COLLECTED:
|
|
520
536
|
reason = "no tests collected"
|
|
521
537
|
return PythonResult(status="none", files=names, reason=reason, output=output, interpreter=interpreter)
|
|
@@ -529,6 +545,8 @@ def _run_python(root: Path, files: list[Path], session: LocalSession) -> PythonR
|
|
|
529
545
|
reason=reason,
|
|
530
546
|
output=output,
|
|
531
547
|
interpreter=interpreter,
|
|
548
|
+
cases=cases,
|
|
549
|
+
cases_omitted=omitted,
|
|
532
550
|
)
|
|
533
551
|
status = "passed" if proc.returncode == 0 else "failed"
|
|
534
552
|
return PythonResult(
|
|
@@ -540,9 +558,63 @@ def _run_python(root: Path, files: list[Path], session: LocalSession) -> PythonR
|
|
|
540
558
|
skipped=skipped,
|
|
541
559
|
output=output,
|
|
542
560
|
interpreter=interpreter,
|
|
561
|
+
cases=cases,
|
|
562
|
+
cases_omitted=omitted,
|
|
543
563
|
)
|
|
544
564
|
|
|
545
565
|
|
|
566
|
+
PROBLEM_LIMIT = 1000
|
|
567
|
+
MAX_PYTHON_CASES = 500
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _junit_cases(path: Path, root: Path) -> list[PythonCase]:
|
|
571
|
+
import xml.etree.ElementTree as ET
|
|
572
|
+
|
|
573
|
+
return [PythonCase.of(case, root) for case in ET.parse(path).iter("testcase") if case.get("name")]
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _bounded(cases: list[PythonCase]) -> tuple[list[PythonCase], int]:
|
|
577
|
+
ordered = sorted(cases, key=lambda case: case.status == "passed")
|
|
578
|
+
return ordered[:MAX_PYTHON_CASES], max(0, len(ordered) - MAX_PYTHON_CASES)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _readable(name: str) -> str:
|
|
582
|
+
base, bracket, params = name.partition("[")
|
|
583
|
+
words = base.removeprefix("test_").replace("_", " ").strip()
|
|
584
|
+
return f"{words} [{params}" if bracket else words
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _case_file(case, root: Path) -> str:
|
|
588
|
+
declared = case.get("file")
|
|
589
|
+
if declared:
|
|
590
|
+
return declared
|
|
591
|
+
module = (case.get("classname") or "").split(".")
|
|
592
|
+
index = next((at for at in range(len(module) - 1, -1, -1) if module[at].startswith("test_") or module[at].endswith("_test")), len(module) - 1)
|
|
593
|
+
dotted = "/".join(module[: index + 1]) + ".py"
|
|
594
|
+
if (root / dotted).is_file():
|
|
595
|
+
return dotted
|
|
596
|
+
found = sorted((root / TESTS_DIR).rglob(f"{module[index]}.py"))
|
|
597
|
+
return str(found[0].relative_to(root)) if found else f"{TESTS_DIR}/{module[index]}.py"
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _case_status(case) -> str:
|
|
601
|
+
outcomes = {child.tag: child for child in case}
|
|
602
|
+
skipped = outcomes.get("skipped")
|
|
603
|
+
if "failure" in outcomes or "error" in outcomes:
|
|
604
|
+
return "failed"
|
|
605
|
+
if skipped is not None and skipped.get("type") == "pytest.xfail":
|
|
606
|
+
return "passed"
|
|
607
|
+
return "skipped" if skipped is not None else "passed"
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _case_problems(case) -> list[str]:
|
|
611
|
+
return [
|
|
612
|
+
(child.get("message") or child.text or child.tag).strip()[:PROBLEM_LIMIT]
|
|
613
|
+
for child in case
|
|
614
|
+
if child.tag in ("failure", "error")
|
|
615
|
+
]
|
|
616
|
+
|
|
617
|
+
|
|
546
618
|
def _junit_counts(path: Path) -> tuple[int, int, int, int] | None:
|
|
547
619
|
if not path.is_file():
|
|
548
620
|
return None
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
(``_bundle.detect_kind``), the data commands (``data.world_kind``), the hub
|
|
5
5
|
client (``harbor_adapter.detect_env_kind``), the benchmark runner
|
|
6
6
|
(``engines/hub_bundle.detect_env_kind``), the platform bridge
|
|
7
|
-
(``platform.world_root``) and the World Host (``
|
|
7
|
+
(``platform.world_root``) and the World Host (``host_versions.world_tree``) all read
|
|
8
8
|
their answer from here. The TypeScript twin is
|
|
9
9
|
``packages/shared/src/world/`` (``WorldTree`` + ``World.fromTree``); the
|
|
10
10
|
corpus under ``conformance/worlds/`` pins both to the same answers.
|
|
@@ -26,6 +26,20 @@ CALL_BODY_LIMIT_LABEL = "4mb"
|
|
|
26
26
|
#: ``session status`` warns once an unpinned session has less than this left.
|
|
27
27
|
IDLE_WARNING_MINUTES = 5
|
|
28
28
|
|
|
29
|
+
#: The most entries one page of the world request ledger holds, as the platform lists it
|
|
30
|
+
#: and as a World Host drains a session's outbox.
|
|
31
|
+
REQUEST_LOG_PAGE_MAX = 200
|
|
32
|
+
|
|
33
|
+
#: How much of each body the request ledger keeps: the first this many bytes of its text.
|
|
34
|
+
REQUEST_BODY_CAPTURE_BYTES = 64 * 1024
|
|
35
|
+
|
|
36
|
+
#: What a World Host's heartbeat names to say it serves open sessions; the platform places an
|
|
37
|
+
#: open session only on a host that does.
|
|
38
|
+
OPEN_DOOR_FEATURE = "open-door"
|
|
39
|
+
|
|
40
|
+
#: The code a session the World Host did not serve answers its request ledger read with.
|
|
41
|
+
REQUESTS_NOT_RECORDED = "requests_not_recorded"
|
|
42
|
+
|
|
29
43
|
#: The task an exploratory session runs on when the caller names none, and
|
|
30
44
|
#: the ``task.source`` the platform reports for it.
|
|
31
45
|
EXPLORATORY_TASK: dict[str, str] = {"instruction": "exploratory session"}
|
|
@@ -93,7 +93,11 @@ ship `connector.toml` in the template directory. `init` copies it to the world r
|
|
|
93
93
|
The format is documented in `SKILL.md` under "Custom connections" and "Serving the vendor's API":
|
|
94
94
|
|
|
95
95
|
- `[connector]` slug and `base_url`; `[auth]` scheme (`header|bearer|basic|query|oauth2|none`),
|
|
96
|
-
`name`, and `secret_env` naming the environment variable, never a value
|
|
96
|
+
`name`, and `secret_env` naming the environment variable, never a value; `prefixes` when the
|
|
97
|
+
vendor takes other `Authorization` words than `Bearer`, `credential = "password"` when its Basic
|
|
98
|
+
key is the password; `[auth.login]` when the vendor trades its key for a token,
|
|
99
|
+
`[auth.downloads]` when it hands out links to files; `[api] unauthorized_status` and
|
|
100
|
+
`unauthorized` as the vendor refuses a bad key.
|
|
97
101
|
- `[capture]` pacing (`requests_per_minute` low; providers ask us to be sparing), cursor
|
|
98
102
|
pagination when the vendor has it (`cursor_field` in the body or `cursor_header`);
|
|
99
103
|
`[redact]` `hash`/`drop` for every PII field the vendor documents returning, `round` for
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: benchmarks-getting-started
|
|
3
|
-
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
3
|
+
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state and the requests it received) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Benchmarks: getting started
|
|
@@ -355,13 +355,12 @@ gateway worlds schema task add ./crm-bench close_ticket --prompt "Close ticket T
|
|
|
355
355
|
gateway worlds schema task add ./crm-bench escalate --prompt "..." --verifier state_present --world slack=state_present
|
|
356
356
|
```
|
|
357
357
|
|
|
358
|
-
- `--prompt` is what the agent is told. `--verifier <name>` names `verifiers/<name>.sql`; it must
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
`
|
|
362
|
-
- A verifier
|
|
363
|
-
|
|
364
|
-
"crm_tickets" WHERE id = 'T-7' AND status = 'closed';`
|
|
358
|
+
- `--prompt` is what the agent is told. `--verifier <name>` names `verifiers/<name>.sql`; it must exist — `schema verifier add <dir> <name> --entity <entity> [--where "<sql condition>"]` scaffolds
|
|
359
|
+
it (without `--where`, the `state_present` placeholder). Each verifier becomes a key in `rewards`; `reward` is their mean.
|
|
360
|
+
- A verifier is one SQL statement returning one number in 0..1 over the world's tables (entity names from `schema/world.json`):
|
|
361
|
+
`SELECT CASE WHEN COUNT(*) > 0 THEN 1.0 ELSE 0.0 END FROM "crm_tickets" WHERE id = 'T-7' AND status = 'closed';`
|
|
362
|
+
- A verifier can also read every request the session's world received, in `gateway_requests` (and its summary `gateway_journal`):
|
|
363
|
+
`SELECT CASE WHEN COUNT(*) = 0 THEN 1.0 ELSE 0.0 END FROM gateway_requests WHERE method = 'DELETE';`. Only a World Host session records them (`dry-run` and `validate` read them as not recorded, and such a verifier scores 0 there whatever it accepts); on a journal that may be missing requests it scores 0 with the reason unless `gateway-env.toml` has `[verifiers.<name>] accepts_incomplete_journal = true`, which suits a check a missing request can only fail (`COUNT(*) > 0`), never `= 0`. A linked world's verifier gets no journal, so one that reads it scores 0. Columns, fail-closed rules, declarative and `verify.py` checks over the same requests: `worlds-getting-started` `references/journal-checks.md`.
|
|
365
364
|
- `--world <alias>=<verifier>` grades a linked world's state too (`schema link` first).
|
|
366
365
|
- Stored task specs (`instruction`, `seed`, `grader` of kind `assertions` | `python` | `rubric`):
|
|
367
366
|
`gateway worlds task create task.json --world <slug>`; `worlds-getting-started` §6.
|
|
@@ -496,5 +495,5 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
496
495
|
|
|
497
496
|
- `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
|
|
498
497
|
- `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
|
|
499
|
-
- Consumer pages: `
|
|
498
|
+
- Consumer pages (docs.surfacearea.ai): `cli/benchmarks`, `worlds/benchmarks`, `evaluation/benchmarks-harbor`.
|
|
500
499
|
- `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
|
|
@@ -68,9 +68,9 @@ real row; captured exports from that tenant become the goldens.
|
|
|
68
68
|
| Slug | State | Notes |
|
|
69
69
|
| --- | --- | --- |
|
|
70
70
|
| slack | package 2.1.1 / domain 2.0.0 / mapping 1, spec-derived fields (wave 1) | base; 34 normalized entities / 72 `$defs` / 27 tools; `connector.toml` serves 56 Web API operations from `api/handlers.py` (bearer `SLACK_BOT_TOKEN`, Slack `ok`/`error` envelopes); `scenarios/default` seeds 34 entities / 84 synthetic rows; `vendor-openapi.json` (trimmed, response schemas relocated to OpenAPI 3 `content`) — `connector conform` 51/54 documented operations pass, 3 fail on channel-keyed `file.shares.public` that the 2020 spec stubs empty, `bookmarks.*` undocumented there. `subset`: no live round trip. Empty init or populated synthetic/selected capture → immutable hosted world; [on-demand fixtures](../../../samples/schema-worlds/README.md#slack-fixtures) |
|
|
71
|
-
| github | spec-derived, wave 2 (domain 4.0.0) | base; pinned GitHub REST OpenAPI (`adb01e9`, 2026-09-15) plus the GraphQL review-state declaration. 45 entities / 5,365 recursive fields / 4 tools / 203 served operations (`connector.toml` + `api/handlers.py`; `scenarios/default` seeds 111 rows across every entity; `vendor-openapi.json` trimmed to the served paths, 203/203 conform). The fullest vendor read form is each entity's root (full-repository, public-user, organization-full, team-full, ...) with the narrower forms' required lists intersected where scope.toml lists several read_forms; route containment and nested-scalar projections are direct fields so keys and relationships are enforceable; many-to-many links are membership entities. Issue state/state-reason come from declared REST sibling value blocks; review state from `PullRequestReviewState`; user `type` stays open. 1,026 documented routes unserved, pagination not emulated; `subset` until a test-org round trip and synthetic goldens exist. |
|
|
71
|
+
| github | spec-derived, wave 2 (domain 4.0.0) | base; pinned GitHub REST OpenAPI (`adb01e9`, 2026-09-15) plus the GraphQL review-state declaration. 45 entities / 5,365 recursive fields / 4 tools / 203 served operations (`connector.toml` + `api/handlers.py`; `scenarios/default` seeds 111 rows across every entity; `vendor-openapi.json` trimmed to the served paths, 203/203 conform). The fullest vendor read form is each entity's root (full-repository, public-user, organization-full, team-full, ...) with the narrower forms' required lists intersected where scope.toml lists several read_forms; route containment and nested-scalar projections are direct fields so keys and relationships are enforceable; many-to-many links are membership entities. Issue state/state-reason come from declared REST sibling value blocks; review state from `PullRequestReviewState`; user `type` stays open. 1,026 documented routes unserved, pagination not emulated; `subset` until a test-org round trip and synthetic goldens exist. Sends GitHub's webhooks (`[webhooks]`, `github` signing): `ping`, `issues`, `issue_comment`, `pull_request`, `push`; JSON only, repository hooks only. |
|
|
72
72
|
| salesforce | docs-derived, wave 2 (full-parity expansion) | tenant-shaped skeleton from 99 pinned Winter '27 (API 68.0, doc 264.0) Object Reference pages plus 32 REST API Developer Guide resource pages: 99 entities / 2,519 top-level fields (2,695 recursive) / 215 restrict relationships / 102 closed enums / 14 unique constraints / 4 tools; `connector.toml` serves 332 operations (Resources by Version, Describe Global, sObject Basic Information, sObject Describe, `/query` SOQL subset, `/search` SOSL subset, and per-object retrieve/create/update/delete exactly where each page's Supported Calls lists them) with `api/handlers.py` for the computed routes; `scenarios/default` seeds 2 synthetic rows in every entity (198 rows). **Describe-on-bind is required** for the org's actual fields, relationships, picklists, custom schema, record types and permissions; the served `/describe` answers from the skeleton and says so. Only Restricted picklists with an explicit vendor value block become enums; org-controlled picklists (StageName, Lead.Status, Case Status/Priority/Origin, ...) remain open strings. Single-target `Refers To` fields become relationships; polymorphic (`WhoId`, `WhatId`, Group-or-User owners: 39) and references to unmodelled objects (183) remain ids. Not served, reported: upsert by external id, deleted/updated, queryAll, composite, limits, Bulk/Tooling/Metadata/Streaming, blob bodies. Conformance not applicable: Salesforce publishes only an org-generated OAS3 beta. `subset`: no org describe, Developer Edition/scratch-org round trip or goldens. |
|
|
73
|
-
| linear | SDL-derived, wave 2 | base; transcribed from the pinned linear/linear `packages/sdk/src/schema.graphql` (commit `23f11eb4`). 32 entities / 600 fields / 114 restrict relationships / 16 enums / 4 tools; 150 REST-shaped `connector.toml` operations (55 read, 24 create, 28 update, 23 delete, 20 handler) with a synthetic `default` scenario for every entity. Vendor camelCase is kept; GraphQL non-null is required, nullable is required-but-null. `Project.state` stays despite `@deprecated` because the brief requires it; `Issue.priority` is integer 0..4 from the description. `linear_roadmaps` has no routes because every roadmap root field is `@deprecated`. OpenAPI conformance is not applicable (GraphQL only, no vendor OpenAPI). `subset`: 90 Node types excluded, 28 generated `<field>Id` names unconfirmed, no live introspection or workspace round trip. |
|
|
73
|
+
| linear | SDL-derived, wave 2 | base; transcribed from the pinned linear/linear `packages/sdk/src/schema.graphql` (commit `23f11eb4`). 32 entities / 600 fields / 114 restrict relationships / 16 enums / 4 tools; 150 REST-shaped `connector.toml` operations (55 read, 24 create, 28 update, 23 delete, 20 handler) with a synthetic `default` scenario for every entity. Vendor camelCase is kept; GraphQL non-null is required, nullable is required-but-null. `Project.state` stays despite `@deprecated` because the brief requires it; `Issue.priority` is integer 0..4 from the description. `linear_roadmaps` has no routes because every roadmap root field is `@deprecated`. OpenAPI conformance is not applicable (GraphQL only, no vendor OpenAPI). `subset`: 90 Node types excluded, 28 generated `<field>Id` names unconfirmed, no live introspection or workspace round trip. Sends Linear's webhooks (`[webhooks]`, `linear` signing) for `Issue` and `Comment` create/update/remove, filtered by `resourceTypes` and team. |
|
|
74
74
|
| jira | spec-derived, schema-worlds pass | mixed; `$defs` transcribed from the pinned Jira Cloud platform REST v3 OpenAPI (1001.0.0-SNAPSHOT, 2,473,385 bytes) and the Jira Software Cloud OpenAPI (1001.0.0) for boards/sprints. 21 entities / 65 `$defs` (477 properties) / 32 relationships / 5 tools / 78 `connector.toml` operations (78/78 conform against the trimmed `vendor-openapi.json`). Issues are keyed by immutable `id` with `key` unique; transitions by (`issueId`, `id`); sprint membership is a link entity. `IssueBean.fields` is `{}` in the vendor document, so an authored `IssueFields` `$def` types 38 system-field members and stays open for `customfield_*` overlay values. Enums only where the vendor declares them (`projectTypeKey`, `accountType`, `style`, `assigneeType`, sprint `state` from the `getAllSprints` filter); `StatusCategory.key` stays open. `scenarios/default` seeds every entity synthetically. `subset`: no Jira Cloud round trip, no goldens, Rovo MCP tool parity unverified. |
|
|
75
75
|
| microsoft-teams | authored base, wave 3 (served) | [Usage, artifacts, scope, and pinned evidence](../../../documentation/features/microsoft-teams-schema.md). 19 entities / 81 vendor `$defs` / 0 tools; teams, channels (incoming, shared-with), chats, chat views, team/channel/chat members, channel messages, replies, chat messages, hosted contents, tags, tag members, pinned messages, async operations. `connector.toml`: 99 Graph v1.0 operations (`auth.scheme = "none"`), all `action = "handler"` because Graph answers carry `@odata.*` annotations and hide the capture-context key column; documented statuses 200/201/202/204/207. `scenarios/default`: 65 synthetic rows. `vendor-openapi.json`: pinned Graph OpenAPI trimmed to the 68 served path items (302 KB) with `2XX`/`204` success keys rewritten to the documented concrete statuses (listed in the sidecar); `connector conform` 98/99, the miss being `DELETE .../incomingChannels/{id}/$ref` which the docs describe but the OpenAPI omits. Not served: user-scoped aliases, `getAllMessages`/delta exports, `$value` binaries, apps/tabs/photo/schedule/permissionGrants. `subset`: no sandbox round trip or goldens. |
|
|
76
76
|
| google-drive | discovery-derived, wave 2 | base; derived from the pinned Drive v3 Discovery document (revision 20260904, sha256 `6d38b6c8e1d1a45fa1b30744523dc6c400ad612997f8e67d112271b573e4948e`), no vendor OpenAPI exists. 10 entities / 34 `$defs` / 410 recursive fields / 4 tools; `drive_folders` is gone (folders are files by `mimeType`). All ten documented resources are entities (Drive, File, Permission, Revision, Comment, Reply, Label, App, AccessProposal, Approval); 49 of 64 documented methods are served from `connector.toml` (changes, channels, about, export/download, CSE tokens, operations and the deprecated `teamdrives.*` are not). Child rows carry route-context `fileId`/`commentId` keys; `parents` stays the vendor array and is not a relationship. Enums are the vendor's declared arrays plus the value lists its descriptions state, recorded separately in `provenance.json`. `vendor-openapi.json` is generated from the Discovery document and trimmed to the served operations; `connector conform` 49/49 against it after seeding `scenarios/default`. `subset`: prose-enum completeness and list-envelope constants unverified, no test-domain round trip or goldens. |
|
|
@@ -80,7 +80,7 @@ real row; captured exports from that tenant become the goldens.
|
|
|
80
80
|
| custom | hand-authored, worked example | the blank starting point for any arbitrary connector, now a worked example of the authoring contract with no vendor behind it: 1 entity (`records`, the only name the scaffold test allows) / 15 fields / 3 tools, showing a primary key, a sparse unique key (`slug`), a self-relationship (`parent_id -> records`, restrict), an enum in `$defs` (`status`), optional (`owner_email`) vs optional-nullable (`parent_id`, `priority`, `due_at`) vs required-nullable (`source.external_id`), a closed nested object and an open bag. Only the original five fields (`id`, `kind`, `name`, `attributes`, `updated_at`) are required and `kind` stays an open string: the SDK test suite uses this template as its blank fixture and inserts rows of that shape, so the example grows by optional fields only. No `connector.toml` ships: the redaction tests need a policy-free world and the same tests create `connector.toml`, `api/` and `scenarios/` themselves and read back an exact template file set, so the template also ships no `api/handlers.py`, no `scenarios/default` seed and no `vendor-openapi.json` (conformance not applicable). Everything is synthetic; `schema init --connector custom` renames the world and drops the tools unless `--with-example-tool`. |
|
|
81
81
|
| workday | hand-authored, wave 3 + REST expansion | tenant-shaped skeleton from pinned public WWS v46.2 WSDL/XSD files (Human Resources, Staffing, Absence Management, Compensation) plus five pinned public Workday REST Swagger 2.0 documents (common v1, staffing v7, absenceManagement v5, compensation v3, person v4, dated 2026-09-12): 27 entities (7 `workday_*` WWS + 20 `workday_rest_*`) / 729 recursive fields / 14 relationships / 3 tools. `connector.toml` serves 62 of the 332 documented REST operations (OAuth2, offset/limit `data[]` envelope, PII redaction, six handler-backed workflow routes); unmodelled routes are listed in `provenance.json`. `scenarios/default` seeds synthetic rows for every entity; `vendor-openapi.json` (OAS 3.0.3 converted and trimmed from the Swagger sources) conforms 62/62. **Describe-on-bind is required** for an actual tenant (tenant WSDL, REST host and OAuth client, custom objects/fields, RaaS). WWS and REST identifier systems stay separate entities until a tenant overlay maps them. Workday-namespaced `wd:enumeration` blocks are used only where the selected reference-ID types declare them; tenant business values remain open. `xsd:date` maps to `format: date`; `xsd:time` remains unformatted because the source permits a missing timezone. `subset`: no tenant round trip, no live capture, timeTracking v6 unreachable. |
|
|
82
82
|
| kandji | hand-authored, wave 3 (expanded 2026-09-15) | base; transcribed from the pinned vendor Postman collection rendered by api-docs.iru.com (135 requests), with the community OpenAPI (`MScottBlake/kandji-openapi@46fa742`) used only as a cross-check. 21 entities / 234 recursive fields / 25 `$defs` / 3 tools; `connector.toml` serves 48 operations (49 of the 135 documented requests: devices, apps, library-item statuses, parameters, commands, activity, notes CRUD, blueprints CRUD + templates + library-item assignment, library-item status/activity, users, tags CRUD, custom apps/profiles/scripts, in-house apps, self-service categories) with 21 handler routes; `scenarios/default` seeds 57 synthetic rows. Device `user` stays as the vendor's nested object-or-empty-string shape; `/user/id` is enforced when assigned and the empty-string sentinel is treated as absent. Nine vendor-declared value blocks (blueprint type, library-item status, activity type, parameter status, integer command status codes, custom-app install type/enforcement, script frequency); platform values are example-derived and completeness remains unverified. Apple-family app shape only; Windows/Android forms need a polymorphic-key model. Trimmed community OpenAPI shipped as `vendor-openapi.json` (18 paths / 26 ops): `connector conform` 16/26 pass; the 10 failures are the community document's inferred types (booleans as integer, arrays as object, nullable `next` as string, blueprint DELETE 200) contradicted by the vendor examples. Not served: device details/Prism, actions, secrets, ADE, threats, vulnerabilities, audit, blueprint routing, upload handshakes. `subset`: no vendor OpenAPI, no account-team sandbox, tenant round trip, or goldens. |
|
|
83
|
-
| jamf | docs-derived, wave 4 | mixed; generated from the OpenAPI 3.0.1 fragments embedded in every pinned developer.jamf.com reference page (Jamf Pro API 11.32.0: 825 pages / 823 operations; Classic API 11.32.0: 689 pages / 613 operations; all fetched 2026-09-15 and hashed in provenance.json) because no per-instance `/api/schema` was available. 66 entities / 166 `$defs` / 2099 recursive fields / 4 tools. Pro resources keep string ids, Classic resources integer ids (root `id` = `general.id`); static and smart groups share one entity per platform; computer and mobile-device group membership are join entities with both foreign keys, prestages and DEP devices point at their device enrollment (7 relationships, all `restrict`); group-name uniqueness is asserted because the create routes document `422 Duplicate group name`, nothing else is. connector.toml serves 302 documented routes (178 Pro under `/api`, 124 Classic under `/JSSResource`; 110 handlers for paged lists, membership and group writes, 90 reads, 25 updates, 77 deletes); the 1134 documented-but-unserved operations are listed per path family in scope.toml `[unserved]`. `vendor-openapi.json` is the two aggregates trimmed to the served routes and their schemas (431 KB; untrimmed hashes in provenance.json) and `connector conform` passes 302/302 after seeding scenarios/default (66 seed files, 198 synthetic rows). Classic list responses follow the pinned array-of-wrapper schema, not the live envelope; Classic XML writes are not
|
|
83
|
+
| jamf | docs-derived, wave 4 | mixed; generated from the OpenAPI 3.0.1 fragments embedded in every pinned developer.jamf.com reference page (Jamf Pro API 11.32.0: 825 pages / 823 operations; Classic API 11.32.0: 689 pages / 613 operations; all fetched 2026-09-15 and hashed in provenance.json) because no per-instance `/api/schema` was available. 66 entities / 166 `$defs` / 2099 recursive fields / 4 tools. Pro resources keep string ids, Classic resources integer ids (root `id` = `general.id`); static and smart groups share one entity per platform; computer and mobile-device group membership are join entities with both foreign keys, prestages and DEP devices point at their device enrollment (7 relationships, all `restrict`); group-name uniqueness is asserted because the create routes document `422 Duplicate group name`, nothing else is. connector.toml serves 302 documented routes (178 Pro under `/api`, 124 Classic under `/JSSResource`; 110 handlers for paged lists, membership and group writes, 90 reads, 25 updates, 77 deletes); the 1134 documented-but-unserved operations are listed per path family in scope.toml `[unserved]`. `vendor-openapi.json` is the two aggregates trimmed to the served routes and their schemas (431 KB; untrimmed hashes in provenance.json) and `connector conform` passes 302/302 after seeding scenarios/default (66 seed files, 198 synthetic rows). Classic list responses follow the pinned array-of-wrapper schema, not the live envelope; Classic XML writes are served only for webhooks (create, update). Sends Jamf's `ComputerAdded` webhook (`[webhooks]`, unsigned) to JSON subscriptions; XML subscriptions and Basic auth are not carried. `subset` until a per-instance schema, a sandbox round trip and goldens exist. |
|
|
84
84
|
| apple-business-manager | docs-derived, wave 3 (full-parity pass) | base; generated from the complete pinned Apple Business API 2.4 DocC JSON corpus (329 pages, per-page sha256 in provenance.json). 24 entities / 281 recursive fields / 28 closed enums / 2 read tools: every documented resource type (OrgDevice, AppleCareCoverage, MdmDevice, MdmDeviceDetail, MdmServer, OrgDeviceActivity, User, UserGroup, OrganizationalUnit, Blueprint, Configuration, App, Package, AuditEvent with all eventData payloads) plus one linkage entity per documented relationship. Vendor `attributes` nesting is preserved; nested `enrolledUserId` and user-group `ouId` relationships are enforced, `roleOuList[].ouId` is not (array traversal); `productFamily` stays open because the shared hr/apple test fixture inserts `MAC`/`IPAD` against Apple's documented `Mac`/`iPad`. `connector.toml` declares all 56 documented routes (OAuth2 client credentials, cursor paging) answered by `api/handlers.py` in Apple's JSON:API envelope with `fields[type]`, `include`, `filter[...]`; `scenarios/default` seeds 69 synthetic rows across every entity. No OpenAPI exists, so `connector conform` is not applicable; the six `DELETE .../relationships/*` routes answer 400 because the world API drops DELETE bodies. `subset`: no sandbox, no live evidence, no goldens. |
|
|
85
85
|
| netsuite | docs-derived, wave 3 (expanded) | tenant-shaped skeleton from the pinned 2026.1 REST API Browser (28 chunks, sha256 in provenance.json; 2023.2 archive as cross-check) plus Oracle record-service, filtering, paging, upsert, error and OAuth docs: 22 standard records as 23 entities / 325 `$defs` / 11,385 recursive fields / 171 relationships / 3 tools / 144 `connector.toml` operations (list, create, get, PATCH, upsert `eid:`, delete per record and 12 modelled-to-modelled `!transform` routes; nsError envelope, `q` filter grammar, `limit`/`offset` paging, bearer required). **Describe-on-bind is required** for role/account-specific records, custom fields, lists, segments, enums, and nullability. Vendor nesting is preserved; `salesOrder-itemElement` is also projected as `netsuite_sales_order_lines` (`order_id`, `line`). Single-target references become restrict relationships; five `one of` references the brief requires (`entity`/`customer` on sales orders, invoices, credit memos, payments; line `item`) are narrowed to customers/items and listed in scope.toml; the other 237 and all collection-item paths stay unenforced. `externalId` uniqueness follows the `eid:` address; the brief's business numbers (`entityId`, `itemId`, `tranId`, `acctNumber`, subsidiary `name`) are declared as the world's contract, not as vendor-schema facts. Default scenario: 97 synthetic rows across all 23 entities. No `vendor-openapi.json`: the Browser is HTML and the OpenAPI 3.0 metadata is per account, so `conform` is not applicable. `suiteql` is not shipped. `subset`: no account metadata overlay, round trip, or goldens. |
|
|
86
86
|
| microsoft-teams | authored schema-only base | [Usage, artifacts, scope, and pinned evidence](../../../documentation/features/microsoft-teams-schema.md); no tools or live integration |
|
|
@@ -75,8 +75,8 @@ registration, a platform builder, live ingestion, tools, or a registry framework
|
|
|
75
75
|
| `gateway worlds schema import PATH --from <kind> --spec <url\|path> [--scope scope.toml]` | not implemented; author the contract and provenance manually |
|
|
76
76
|
| `gateway worlds schema parity PATH --surface <tools-list.json> \| --connection <id>` (advisory report) | not implemented |
|
|
77
77
|
| `gateway worlds playground <connector>` | not implemented |
|
|
78
|
-
| Platform upload of a JSON Schema, OpenAPI, or DDL as a new connector | not implemented as a UI upload; the platform agent
|
|
79
|
-
| Platform agent record ingestion
|
|
78
|
+
| Platform upload of a JSON Schema, OpenAPI, or DDL as a new connector | not implemented as a UI upload; the platform agent authors the tree in its sandbox and runs `gateway worlds create <slug> --from ./dir` through `run_gateway_cli`, the same door as the CLI (2026-10-10) |
|
|
79
|
+
| Platform agent record ingestion | the CLI's own `data import <slug>`, `pull`, `push` and `describe` through `run_gateway_cli`; see `documentation/features/agent.md` |
|
|
80
80
|
| SDK/public and platform publication | validates assembled files and actual stored archive bytes through the trusted compiler/materializer, including READY/dedup/ref paths |
|
|
81
81
|
| Connectors Slack world creation | durable structured capture → shared Python mapper → immutable publication; explicit synthetic mode |
|
|
82
82
|
| Worker schema tests/workbench | shared validation and isolated E2B author execution; missing isolation is non-green |
|
|
@@ -143,7 +143,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
143
143
|
Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
|
|
144
144
|
exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
|
|
145
145
|
with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
|
|
146
|
-
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
|
|
146
|
+
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; `raise Refusal(404, body)` (imported from `gateway_env` at module top) answers that 4xx and body; anything else a handler raises is a
|
|
147
147
|
500 `{error: "engine", detail, handler}`; never a bare exception string). A route's handler answers the
|
|
148
148
|
same way on every door, and an answer JSON cannot carry (a `datetime`, a set, NaN) is its 400; a route
|
|
149
149
|
answering bytes names its `Content-Type` (a vendor's `404 text/plain`), and the bytes go out as sent.
|
|
@@ -194,10 +194,11 @@ No `upsert`: `get`, then `insert` or `update`. Read-only SQL: `gateway_env.db.on
|
|
|
194
194
|
## Runtime authoring boundary
|
|
195
195
|
|
|
196
196
|
When adding a bridge command to `schema/platform.py`, mirror it as a typed wrapper in
|
|
197
|
-
`worldSchemaRuntime.ts` (`compileWorldFiles` / `
|
|
198
|
-
in, `{entries,
|
|
199
|
-
`tests/environment/test_schema_platform.py`, and let
|
|
200
|
-
the subprocess
|
|
197
|
+
`worldSchemaRuntime.ts` (`compileWorldFiles` / `importWorldData` are the pattern: file map
|
|
198
|
+
in, `{entries, ...}` out, errors as `SchemaBridgeError`), cover it in
|
|
199
|
+
`tests/environment/test_schema_platform.py`, and let a service call the wrapper, never
|
|
200
|
+
the subprocess; the public `/api/public/worlds/runtime` route reaches only the commands in
|
|
201
|
+
`WORLD_RUNTIME_COMMANDS`. The bridge needs Python ≥ 3.12 (`tomllib`, and the store's sqlite `setconfig`); set `GATEWAY_SCHEMA_PYTHON`
|
|
201
202
|
when the default interpreter is older.
|
|
202
203
|
|
|
203
204
|
Use `worldSchemaRuntime.ts` for platform validation and `schema.platform` for
|
|
@@ -273,8 +274,12 @@ These source checks do not imply live-provider validation or executed fidelity s
|
|
|
273
274
|
|
|
274
275
|
A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
|
|
275
276
|
auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
|
|
276
|
-
environment variable (
|
|
277
|
-
|
|
277
|
+
environment variable (a `bearer`'s `prefixes` lists the `Authorization` words the vendor takes,
|
|
278
|
+
`["Bearer", "token"]` for GitHub, the first carried; a `basic` key is the Basic user with an empty
|
|
279
|
+
password, or with `credential = "password"` the password under any user, as Atlassian and
|
|
280
|
+
ServiceNow take it), the vendor's own login route (`[auth.login]`; an oauth2 `token_url` path may
|
|
281
|
+
name a segment, `/ccx/oauth2/{tenant}/token`) and presigned download routes (`[auth.downloads]`),
|
|
282
|
+
capture pacing, a redaction policy (`hash` or `drop` fields at any
|
|
278
283
|
depth), and the operations the agent actually calls (method, path with `{args}`, query
|
|
279
284
|
params, and the entity plus results path their rows land in). The same file is meant to
|
|
280
285
|
drive the platform's custom source later, so describe once.
|
|
@@ -316,6 +321,10 @@ Three refinements the first three vendors forced:
|
|
|
316
321
|
as well as a path argument; a parameter the caller leaves out does not narrow, and one
|
|
317
322
|
sent more than once (`status[]=a&status[]=b`) matches rows carrying any of its values.
|
|
318
323
|
Handlers read every value from `ctx.query_all`; the `query` argument keeps the first.
|
|
324
|
+
- **Uploads.** A multipart form reaches a handler as its fields (a file part reads
|
|
325
|
+
`{"filename", "content_type", "size"}`) with the bytes in `ctx.files[name].content`; a raw
|
|
326
|
+
body (`application/octet-stream`, `text/csv`) reads `body` as `{}` with the bytes in
|
|
327
|
+
`ctx.raw`. A declared route refuses an upload as the vendor's 400.
|
|
319
328
|
|
|
320
329
|
Four more the Brex retrospective forced (each cost a day of one-field-per-run fixing):
|
|
321
330
|
|
|
@@ -349,21 +358,17 @@ redacted pages back into `captures/`.
|
|
|
349
358
|
|
|
350
359
|
### From the platform agent (no checkout)
|
|
351
360
|
|
|
352
|
-
The console agent
|
|
353
|
-
`
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
(
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
refused before anything is queued. `run_gateway_cli` runs the TypeScript `gateway worlds …` CLI
|
|
364
|
-
(the same flags as this page) staged into the sandbox, local-only (paths under `/workspace`,
|
|
365
|
-
no `--host`/`--remote`/`workspace:<slug>`, platform env stripped, pinned to its local runtime),
|
|
366
|
-
so publishing always goes through the tools above. To parallelise several connectors, `spawn_subagent` with a brief
|
|
361
|
+
The console agent runs the CLI on this page through `run_gateway_cli` in its sandbox: write
|
|
362
|
+
`connector.toml` + `schema/` under `./acme` → `connector validate ./acme`, `connector conform ./acme --openapi ./spec.json`,
|
|
363
|
+
`schema check ./acme`, `test ./acme` → `create acme --from ./acme` (billed: the session's
|
|
364
|
+
`provisionWorlds` or `confirm: true`) → `start_connector_capture({ containerId, op, args, params })` →
|
|
365
|
+
`get_connector_capture({ …, downloadTo })` → `connector ingest ./acme` → `push ./acme`, then
|
|
366
|
+
`edit close acme`. `connector publish ./acme` makes it a workspace connector (`hub:write` and
|
|
367
|
+
`allowConnectorWrites`, else `confirm: true`). A `./dir` runs in the sandbox; a slug reaches the
|
|
368
|
+
platform as the user through a key minted for that one call, valid only on the routes that verb calls and
|
|
369
|
+
deleted when the call ends; `--host` and key flags are refused. `list_world_secrets` shows the
|
|
370
|
+
env var names the world holds, never values; a `connector.toml` with an inline credential is
|
|
371
|
+
refused before anything is queued. To parallelise several connectors, `spawn_subagent` with a brief
|
|
367
372
|
naming only the tools that child needs (`allowSubagents` on the session).
|
|
368
373
|
|
|
369
374
|
### From the CLI sandbox (no checkout, no Python)
|
|
@@ -399,23 +404,40 @@ them:
|
|
|
399
404
|
- `unauthorized` / `not_found` / `invalid` are JSON strings replayed on 401 / 404 / 400, so
|
|
400
405
|
the vendor's own error bodies come back; `""` (or `'null'`) means no body at all. A route
|
|
401
406
|
whose missing row the vendor answers its own way sets `not_found` and `not_found_status`
|
|
402
|
-
under `[operations.api]`, falling back to `[api]`.
|
|
407
|
+
under `[operations.api]`, falling back to `[api]`. A path the world does not serve answers
|
|
408
|
+
`not_found`; a path it serves, called with another method, answers `wrong_method` (405 with
|
|
409
|
+
`Allow`, no body unless declared; `wrong_method_status` for a vendor that says 404). A write
|
|
410
|
+
the contract refuses answers `invalid` exactly as declared; the refusal's rule goes to the
|
|
411
|
+
host's log, never the answer.
|
|
412
|
+
- `link_header = true` sends RFC 8288 `Link: <next>; rel="next"` (and `rel="prev"`) on list
|
|
413
|
+
answers, each URL the request's own (GitHub's paging); `cors = true` answers a browser's
|
|
414
|
+
preflight before any credential and names any origin (exposing `Link`) on every answer, for a vendor whose
|
|
415
|
+
API serves pages (`worlds serve` answers only this machine's pages). HEAD answers as GET
|
|
416
|
+
without a body on every world.
|
|
403
417
|
- `[operations.api] action` makes a route a `create`, `update`, `delete` or `handler`; handlers live in the world's `api/handlers.py` and carry computed results and workflow state (`ctx.ladder` on the clock). A world is read _and_ write: an agent that creates a search and gets 501 is not testing anything. Full reference: `documentation/features/connector-worlds.md`.
|
|
404
418
|
- `[operations.api] filter = { email = "attributes.email" }` maps a path argument to the
|
|
405
419
|
row field it selects on, matched case-insensitively at any depth. An operation without
|
|
406
420
|
an `entity` answers 501 `not_modelled` rather than inventing a response.
|
|
421
|
+
- `[auth.login]` mirrors a vendor that trades its key for a token (`path`, `credential = { json |
|
|
422
|
+
form | header = "<field>" }`, `required`, `reply = '{"data": {"token": "{token}"}}'`);
|
|
423
|
+
later calls carry the token the way `scheme` says, and only tokens minted for that session count.
|
|
424
|
+
`[auth.downloads] routes = ["download_x"]` lets those GET routes answer a link a handler signs with
|
|
425
|
+
`ctx.signed_url(path)` without the key. Tokens and links live as long as the session; set
|
|
426
|
+
`ttl_seconds` (login, downloads, oauth2) only when the eval needs expiry. `mode = "open"` under
|
|
427
|
+
`[auth]` opens sessions with no credential at all (a capability URL; `--auth` on the session's
|
|
428
|
+
open overrides; `scheme = "open"` is refused). Full grammar: connector-worlds.md, "The door".
|
|
407
429
|
- `[ui] static = "ui/static"` (+ `entry`, `api_path`, `generated`) is the page over the
|
|
408
430
|
routes: generated by `compile` from the contract unless `generated = false`, served by the
|
|
409
431
|
host on the session's ui origin (`?token=` once, then a cookie; `api_path` rewritten onto
|
|
410
432
|
the routes) and locally on its own port (and at `/__ui/`). A page holds no key; the host presents the world's
|
|
411
|
-
credential for it, the way it does for a
|
|
433
|
+
credential for it, the way it does for a caller holding the session token in `X-World-Session`.
|
|
412
434
|
|
|
413
435
|
`gateway worlds serve PATH --port 8080 [--api-key K]` runs it locally; the runner's admin
|
|
414
436
|
endpoints (`/__world/seed`, `/__world/reset`, `/__world/state`) sit beside the vendor routes
|
|
415
437
|
and answer only the run's `X-World-Admin-Token` (`gateway worlds test` hands a world's pytest
|
|
416
438
|
files theirs as `WORLD_ADMIN_TOKEN`; by hand, use `/__session/*`); `/healthz` answers everyone,
|
|
417
|
-
and a World Host session's doors answer none of the
|
|
439
|
+
and a World Host session's api and tool doors answer none of them (there `/healthz` is the world's own path; its ui origin answers `/healthz` under `api_path` to the twin's own page). `gateway bench push PATH` publishes the world: for a world carrying
|
|
418
440
|
`connector.toml` it packages the api-kind bundle the platform hosts on the way (root
|
|
419
441
|
legacy-shaped on purpose, since the api runner refuses a schema bundle there; `server/`
|
|
420
442
|
needs nothing installed). Open a session with `--surface api` and hand the client that
|
|
421
|
-
URL; the vendor's auth
|
|
443
|
+
URL; the vendor's auth applies as `[auth]` declares it, with the session's own api token as the credential.
|