@withgateway/sdk 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +24 -8
  2. package/dist/cli.js +15468 -11263
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +15233 -11028
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +291 -40
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +284 -37
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +203 -47
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +198 -42
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
  19. package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
  20. package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
  21. package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
  22. package/dist/index.d.mts +3 -3
  23. package/dist/index.d.ts +3 -3
  24. package/dist/index.js +2149 -615
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2053 -566
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/traces/index.js +268 -31
  29. package/dist/traces/index.js.map +1 -1
  30. package/dist/traces/index.mjs +262 -28
  31. package/dist/traces/index.mjs.map +1 -1
  32. package/dist/tracing/index.js +19 -11
  33. package/dist/tracing/index.js.map +1 -1
  34. package/dist/tracing/index.mjs +19 -11
  35. package/dist/tracing/index.mjs.map +1 -1
  36. package/dist/worlds/index.d.mts +886 -117
  37. package/dist/worlds/index.d.ts +886 -117
  38. package/dist/worlds/index.js +1770 -452
  39. package/dist/worlds/index.js.map +1 -1
  40. package/dist/worlds/index.mjs +1752 -434
  41. package/dist/worlds/index.mjs.map +1 -1
  42. package/package.json +14 -10
  43. package/runtime/gatewaysdk/environment/_toml.py +3 -10
  44. package/runtime/gatewaysdk/environment/_world.py +95 -28
  45. package/runtime/gatewaysdk/environment/runtime.py +10 -6
  46. package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
  47. package/runtime/gatewaysdk/environment/schema/api.py +19 -23
  48. package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
  49. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
  50. package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
  51. package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
  52. package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
  53. package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
  54. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
  55. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
  56. package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
  57. package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
  58. package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
  59. package/runtime/gatewaysdk/environment/schema/data.py +21 -5
  60. package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
  61. package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
  62. package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
  63. package/runtime/gatewaysdk/environment/schema/host.py +68 -115
  64. package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
  65. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
  66. package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
  67. package/runtime/gatewaysdk/environment/schema/local.py +7 -10
  68. package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
  69. package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
  70. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
  71. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
  72. package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
  73. package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
  74. package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
  75. package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
  76. package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
  77. package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
  78. package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
  79. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
  80. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
  81. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
  82. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
  83. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
  84. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
  85. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  86. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
  87. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  88. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
  89. package/runtime/gatewaysdk/environment/schema/store.py +15 -6
  90. package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
  91. package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
  92. package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
  93. package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
  94. package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
  95. package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
  96. package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
  97. package/runtime/gatewaysdk/environment/world.py +42 -3
  98. package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
  99. package/skills/benchmarks-getting-started/SKILL.md +70 -22
  100. package/skills/connector-schema-authoring/SKILL.md +54 -4
  101. package/skills/connector-schema-authoring/connector-worlds.md +21 -7
  102. package/skills/connector-schema-authoring/world-schema.md +144 -83
  103. package/skills/hooks-getting-started/SKILL.md +27 -2
  104. package/skills/world-data-ingestion/ROWS.md +62 -0
  105. package/skills/world-data-ingestion/SKILL.md +91 -54
  106. package/skills/worlds-getting-started/SKILL.md +146 -146
  107. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  108. package/skills/worlds-getting-started/references/gotchas.md +30 -0
  109. package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  110. package/skills/worlds-getting-started/references/openapi-import.md +90 -0
@@ -0,0 +1,264 @@
1
+ """Text cells typed by the entity's schema — the one rule both CLIs apply to a CSV/TSV cell.
2
+
3
+ The TypeScript twin is ``worldData/cast.ts``; ``tests/conformance/csv_cast.json`` pins
4
+ them to the same answers. ``integer`` and ``number`` parse as numbers (a number is
5
+ always a float), ``boolean`` takes true/false/1/0 in any case, ``array`` and
6
+ ``object`` parse as JSON of that shape, ``string`` is the text as given. An empty
7
+ cell is ``None`` when the field's type admits null, left out when the field is
8
+ optional, the empty string for a required string, and refused for any other
9
+ required field. A union type (``integer | string``) tries each member in order,
10
+ ``string`` last. A cell that fits none is refused, naming the field and the cell;
11
+ a column the schema does not name stays text (the gate refuses it, as it always did).
12
+
13
+ Standard library only, so the bridge and the api bundle can load it.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import math
20
+ import re
21
+ from collections.abc import Callable, Iterable
22
+ from dataclasses import dataclass
23
+ from pathlib import Path
24
+ from typing import Any
25
+
26
+ from .validation import fail
27
+
28
+ INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
29
+ NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
30
+ BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
31
+ MEMBERS = ("integer", "number", "boolean", "array", "object", "string", "null")
32
+
33
+ _MISSING = object()
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class FieldType:
38
+ """One field as ``gateway worlds describe`` reports it: ``type`` is the JSON Schema
39
+ type word(s) joined by ``" | "``, a ``$ref`` name, or None when the schema says nothing."""
40
+
41
+ name: str
42
+ type: str | None
43
+ required: bool
44
+
45
+
46
+ def _article(kind: str) -> str:
47
+ return "an" if kind[:1].lower() in "aeiou" else "a"
48
+
49
+
50
+ def refuse(where: str, field: str, cell: str | None, kind: str) -> None:
51
+ """The row-level refusal a cell that will not cast earns (same words as the TypeScript twin)."""
52
+ if cell is None:
53
+ fail("rows", f"{where}: {field} is empty; the field is a required {kind}", where)
54
+ fail(
55
+ "rows",
56
+ f"{where}: {field} = {json.dumps(cell)} is not {_article(kind)} {kind}",
57
+ where,
58
+ )
59
+
60
+
61
+ def type_members(kind: str | None) -> list[str]:
62
+ """The typed members of a field's ``type`` (``"integer | null"`` -> integer, null); an
63
+ unknown word (a ``$ref`` name) is dropped."""
64
+ if kind is None:
65
+ return []
66
+ parts = (part.strip().lower() for part in kind.split("|"))
67
+ return [part for part in parts if part in MEMBERS]
68
+
69
+
70
+ def _integer(text: str) -> Any:
71
+ return int(text) if INTEGER_TEXT.match(text) else _MISSING
72
+
73
+
74
+ def _number(text: str) -> Any:
75
+ if not NUMBER_TEXT.match(text):
76
+ return _MISSING
77
+ value = float(text)
78
+ return value if math.isfinite(value) else _MISSING
79
+
80
+
81
+ def _boolean(text: str) -> Any:
82
+ return BOOLEANS.get(text.lower(), _MISSING)
83
+
84
+
85
+ def _json_of(shape: type) -> Callable[[str], Any]:
86
+ def cast(text: str) -> Any:
87
+ try:
88
+ value = json.loads(text)
89
+ except ValueError:
90
+ return _MISSING
91
+ return value if isinstance(value, shape) else _MISSING
92
+
93
+ return cast
94
+
95
+
96
+ def _no_cast(_text: str) -> Any:
97
+ return _MISSING
98
+
99
+
100
+ _CAST: dict[str, Callable[[str], Any]] = {
101
+ "integer": _integer,
102
+ "number": _number,
103
+ "boolean": _boolean,
104
+ "array": _json_of(list),
105
+ "object": _json_of(dict),
106
+ }
107
+
108
+
109
+ class FieldCaster:
110
+ """How one field's cells cast."""
111
+
112
+ def __init__(self, name: str, kind: str | None, required: bool):
113
+ self.name = name
114
+ self.type = kind
115
+ self.required = required
116
+ self.members = type_members(kind)
117
+
118
+ def cast(self, cell: str | None, where: str) -> Any:
119
+ """The typed value; :data:`OMIT` to leave the field out of the row."""
120
+ if cell is None:
121
+ return self._empty(where)
122
+ if not self.members:
123
+ return cell
124
+ text = cell.strip()
125
+ for member in self.members:
126
+ value = _CAST.get(member, _no_cast)(text)
127
+ if value is not _MISSING:
128
+ return value
129
+ if "string" in self.members:
130
+ return cell
131
+ refuse(where, self.name, cell, self.type or "value")
132
+
133
+ def _empty(self, where: str) -> Any:
134
+ if not self.members or "null" in self.members:
135
+ return None
136
+ if not self.required:
137
+ return OMIT
138
+ if "string" in self.members:
139
+ return ""
140
+ refuse(where, self.name, None, self.type or "value")
141
+
142
+
143
+ #: What :meth:`FieldCaster.cast` answers for an empty optional cell: the field stays out of the row.
144
+ OMIT = object()
145
+
146
+
147
+ class RowCaster:
148
+ """The casters of one entity's fields; a column the entity does not name stays text."""
149
+
150
+ def __init__(self, fields: Iterable[FieldType]):
151
+ self._fields = {f.name: FieldCaster(f.name, f.type, f.required) for f in fields}
152
+
153
+ def row(self, record: dict[str, str | None], where: str) -> dict:
154
+ row: dict = {}
155
+ for name, cell in record.items():
156
+ caster = self._fields.get(name)
157
+ value = cell if caster is None else caster.cast(cell, where)
158
+ if value is not OMIT:
159
+ row[name] = value
160
+ return row
161
+
162
+
163
+ #: Text cells as the untyped reader always made them: the text, or None for an empty cell.
164
+ TEXT_CASTER = RowCaster([])
165
+
166
+ EntityCasters = Callable[[str], "RowCaster | None"]
167
+
168
+
169
+ def casters_of(entities: dict[str, list[FieldType]]) -> EntityCasters:
170
+ """Casters by entity name."""
171
+ by_name = {name: RowCaster(fields) for name, fields in entities.items()}
172
+ return by_name.get
173
+
174
+
175
+ def _definition(world: dict, entity: Any) -> dict:
176
+ schema = entity.get("schema") if isinstance(entity, dict) else None
177
+ ref = schema.get("$ref") if isinstance(schema, dict) else None
178
+ if isinstance(ref, str) and ref.startswith("#/$defs/"):
179
+ definition = (world.get("$defs") or {}).get(ref[len("#/$defs/") :])
180
+ return definition if isinstance(definition, dict) else {}
181
+ # An inline payload (`schema entity add --schema '{…}'`) declares its fields right here.
182
+ if isinstance(schema, dict) and isinstance(schema.get("properties"), dict):
183
+ return schema
184
+ return {}
185
+
186
+
187
+ def _field_type(field_schema: Any) -> str | None:
188
+ if not isinstance(field_schema, dict):
189
+ return None
190
+ kind = field_schema.get("type")
191
+ if isinstance(kind, list):
192
+ # A nullable field is `["string", "null"]` in JSON Schema; one readable type, not the list.
193
+ return " | ".join(str(part) for part in kind)
194
+ if kind is None and isinstance(field_schema.get("$ref"), str):
195
+ return field_schema["$ref"].rsplit("/", 1)[-1]
196
+ return kind if isinstance(kind, str) else None
197
+
198
+
199
+ def world_fields(world: dict) -> dict[str, list[FieldType]]:
200
+ """Every entity's fields as a world.json document declares them (the ``$defs`` entry its
201
+ ``schema.$ref`` names, or an inline ``properties``), sorted by entity name — the field
202
+ list ``gateway worlds describe`` reports and a CSV import casts by."""
203
+ gateway = world.get("x-gateway") if isinstance(world, dict) else None
204
+ entities = (gateway.get("entities") or {}) if isinstance(gateway, dict) else {}
205
+ fields: dict[str, list[FieldType]] = {}
206
+ for name, entity in sorted(entities.items()):
207
+ definition = _definition(world, entity)
208
+ required = set(definition.get("required") or [])
209
+ properties = definition.get("properties") or {}
210
+ fields[name] = [
211
+ FieldType(field_name, _field_type(field_schema), field_name in required)
212
+ for field_name, field_schema in properties.items()
213
+ ]
214
+ return fields
215
+
216
+
217
+ def world_casters(world: dict) -> EntityCasters:
218
+ """:func:`casters_of` for a world.json document."""
219
+ return casters_of(world_fields(world))
220
+
221
+
222
+ def schema_fields(schema: Any) -> list[FieldType]:
223
+ """The fields of one resolved entity schema (``properties`` + ``required``)."""
224
+ if not isinstance(schema, dict):
225
+ return []
226
+ required = set(schema.get("required") or [])
227
+ return [
228
+ FieldType(name, _field_type(field_schema), name in required)
229
+ for name, field_schema in (schema.get("properties") or {}).items()
230
+ ]
231
+
232
+
233
+ def contract_casters(entities: dict) -> EntityCasters:
234
+ """:func:`casters_of` for a compiled contract's ``entities`` (each carries its resolved
235
+ ``schema``) — the local twin of :func:`world_casters`."""
236
+ return casters_of({name: schema_fields(entity.get("schema")) for name, entity in entities.items()})
237
+
238
+
239
+ def tree_casters(root: str | Path) -> EntityCasters:
240
+ """:func:`contract_casters` for a working tree: what a ``.csv``/``.tsv`` imported into the
241
+ directory is typed by. A tree that does not compile raises the compiler's own ``ContractError``."""
242
+ from .compiler import compile_bundle
243
+
244
+ return contract_casters(compile_bundle(root, snapshots=False).entities)
245
+
246
+
247
+ def described_casters(described: dict) -> EntityCasters:
248
+ """:func:`casters_of` for what ``GET /api/public/worlds/{slug}`` (``gateway worlds describe``)
249
+ answers: ``entities: [{name, fields: [{name, type, required}]}]`` — the hosted twin of
250
+ :func:`contract_casters`, the same fields the TypeScript CLI casts by."""
251
+ entities: dict[str, list[FieldType]] = {}
252
+ for entity in described.get("entities") or []:
253
+ if not isinstance(entity, dict) or not isinstance(entity.get("name"), str):
254
+ continue
255
+ entities[entity["name"]] = [
256
+ FieldType(
257
+ str(field.get("name")),
258
+ field.get("type") if isinstance(field.get("type"), str) else None,
259
+ bool(field.get("required")),
260
+ )
261
+ for field in entity.get("fields") or []
262
+ if isinstance(field, dict) and isinstance(field.get("name"), str)
263
+ ]
264
+ return casters_of(entities)
@@ -11,6 +11,7 @@ from typing import Any
11
11
  from .._toml import tomllib
12
12
  from .validation import (
13
13
  LANGUAGE,
14
+ MAX_ENTITY_NAME,
14
15
  ContractError,
15
16
  Resolver,
16
17
  canonical,
@@ -59,6 +60,13 @@ def identifier(value: Any, location: str) -> str:
59
60
  "Use a SQL-safe name; _, sqlite_, gateway_, benchmark_ prefixes are reserved",
60
61
  location,
61
62
  )
63
+ if len(value) > MAX_ENTITY_NAME:
64
+ fail(
65
+ "identity",
66
+ f"entity name must be at most {MAX_ENTITY_NAME} characters, got {len(value)}",
67
+ location,
68
+ limit=MAX_ENTITY_NAME,
69
+ )
62
70
  return value
63
71
 
64
72
 
@@ -244,6 +252,9 @@ class Contract:
244
252
  metadata: dict
245
253
  artifacts: dict[str, bytes]
246
254
  lock: dict
255
+ #: Structured, non-fatal findings (``recursive_reference``: a $defs cycle no
256
+ #: entity reaches); never part of the lock or the artifacts.
257
+ warnings: tuple[dict, ...] = ()
247
258
 
248
259
  @property
249
260
  def target(self) -> dict:
@@ -263,7 +274,12 @@ def build(bundle: Path) -> Contract:
263
274
  manifest = bundle_mode(bundle)
264
275
  if manifest is None:
265
276
  fail("language", "Not a schema-backed bundle; declare [schema] language")
277
+ from .cycles import ReferenceCycles
278
+ from .route_bindings import check_route_bindings
279
+
266
280
  resolver = Resolver(bundle)
281
+ cycles = ReferenceCycles(resolver)
282
+ warnings: list[dict] = []
267
283
  document = resolver.load("world.json")
268
284
  annotation = document.get("x-gateway")
269
285
  if not isinstance(annotation, dict):
@@ -312,6 +328,7 @@ def build(bundle: Path) -> Contract:
312
328
  location,
313
329
  )
314
330
  payload = resolver.resolve(entity["schema"], "world.json", f"{location}/schema")
331
+ warnings.extend(cycles.reached(name, entity["schema"], f"{location}/schema"))
315
332
  if (
316
333
  payload.get("type") != "object"
317
334
  or payload.get("additionalProperties") is not False
@@ -382,6 +399,7 @@ def build(bundle: Path) -> Contract:
382
399
  "relationships": sorted(relationships, key=canonical),
383
400
  "location": location,
384
401
  }
402
+ warnings.extend(cycles.unreached(document.get("$defs", {}), "world.json"))
385
403
  for name, entity in compiled.items():
386
404
  for rel in entity["relationships"]:
387
405
  location = f"{entity['location']}/relationships"
@@ -524,7 +542,9 @@ def build(bundle: Path) -> Contract:
524
542
  "tools_hash": digest(encoded(compiled_tools)),
525
543
  "artifacts": {p: digest(b) for p, b in sorted(artifacts.items())},
526
544
  }
527
- return Contract(metadata, artifacts, lock)
545
+ contract = Contract(metadata, artifacts, lock, tuple(warnings))
546
+ check_route_bindings(bundle, contract)
547
+ return contract
528
548
 
529
549
 
530
550
  SCALARS = {"string", "integer", "number", "boolean"}
@@ -609,6 +629,7 @@ def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
609
629
  (`gateway worlds schema compile` reports it at once, see ``compile_world``)."""
610
630
  bundle = Path(bundle)
611
631
  contract = build(bundle)
632
+ bind_connection(bundle, contract)
612
633
  for path, content in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
613
634
  destination = bundle / path
614
635
  destination.parent.mkdir(parents=True, exist_ok=True)
@@ -694,12 +715,25 @@ def compile_world(bundle: str | Path) -> tuple[Contract, dict]:
694
715
  return contract, build_all(bundle, contract)
695
716
 
696
717
 
718
+ def bind_connection(bundle: Path, contract: Contract) -> None:
719
+ """A world with a connector.toml: every route lands on an entity the contract
720
+ declares (:meth:`Spec.check_entities`). Runs before compile writes and before
721
+ check trusts the artifacts, so a ghost entity is refused, never served."""
722
+ spec_path = bundle / "connector.toml"
723
+ if not spec_path.is_file():
724
+ return
725
+ from .connector import parse_spec
726
+
727
+ parse_spec(spec_path.read_text(encoding="utf-8")).check_entities(contract.entities)
728
+
729
+
697
730
  def check_bundle(bundle: str | Path) -> Contract:
698
731
  """The contract, provided every generated artifact and every snapshot present in
699
732
  the tree matches it; a snapshot's sidecar must name this contract and the
700
733
  file's own sha256. Absent snapshots are not required: the legacy path serves."""
701
734
  bundle = Path(bundle)
702
735
  contract = build(bundle)
736
+ bind_connection(bundle, contract)
703
737
  for path, expected in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
704
738
  try:
705
739
  matches = (bundle / path).read_bytes() == expected
@@ -48,20 +48,38 @@ PIN = "conform"
48
48
 
49
49
 
50
50
  def load_openapi(path: str | Path) -> dict:
51
- text = Path(path).read_text(encoding="utf-8")
52
- if str(path).endswith((".yaml", ".yml")):
53
- try:
54
- import yaml # optional at runtime; the SDK depends on it, the vendored server does not
55
- except ImportError: # pragma: no cover
56
- fail("conform", "Reading a YAML document needs PyYAML; convert it to JSON", "/conform/openapi")
57
- document = yaml.safe_load(text)
58
- else:
59
- document = json.loads(text)
51
+ return parse_openapi(Path(path).read_text(encoding="utf-8"), str(path))
52
+
53
+
54
+ def parse_openapi(text: str, name: str, *, location: str = "/conform/openapi") -> dict:
55
+ """The one OpenAPI reader: YAML when the name says so or the text is not JSON,
56
+ JSON otherwise. ``conform`` and ``schema init --from-openapi`` both read through
57
+ here, so a document parses the same way wherever it came from."""
58
+ is_json = name.endswith(".json") or (not name.endswith((".yaml", ".yml")) and text.lstrip()[:1] in ("{", "["))
59
+ document = _parse_json(text, name, location) if is_json else _parse_yaml(text, name, location)
60
60
  if not isinstance(document, dict) or "paths" not in document:
61
- fail("conform", f"{path} is not an OpenAPI document (no paths)", "/conform/openapi")
61
+ fail("conform", f"{name} is not an OpenAPI document (no paths)", location)
62
62
  return document
63
63
 
64
64
 
65
+ def _parse_json(text: str, name: str, location: str) -> Any:
66
+ try:
67
+ return json.loads(text)
68
+ except ValueError as error:
69
+ fail("conform", f"{name} is not a JSON document: {error}", location)
70
+
71
+
72
+ def _parse_yaml(text: str, name: str, location: str) -> Any:
73
+ try:
74
+ import yaml # optional at runtime; the SDK depends on it, the vendored server does not
75
+ except ImportError: # pragma: no cover
76
+ fail("conform", "Reading a YAML document needs PyYAML; convert it to JSON", location)
77
+ try:
78
+ return yaml.safe_load(text)
79
+ except yaml.YAMLError as error:
80
+ fail("conform", f"{name} is not a YAML document: {error}", location)
81
+
82
+
65
83
  def server_prefixes(document: dict) -> list[str]:
66
84
  """The path part of each server URL, longest first, so a world whose paths carry
67
85
  the vendor's prefix (``/investigations-v2/breach/...``) still lines up."""
@@ -72,13 +72,12 @@ import urllib.request
72
72
  from dataclasses import dataclass, field
73
73
  from datetime import datetime, timezone
74
74
  from pathlib import Path
75
- from typing import Any, Callable
75
+ from typing import Any, Callable, Iterable
76
76
 
77
- from .._toml import tomllib
78
77
  from .compiler import check_bundle, compile_bundle
79
78
  from .store import SQLiteStore
80
79
  from .transforms import apply_transforms
81
- from .validation import ContractError, fail
80
+ from .validation import ContractError, fail, operation_name, parse_toml
82
81
 
83
82
  SPEC = "connector.toml"
84
83
  CAPTURES = "captures"
@@ -87,7 +86,6 @@ SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
87
86
  DEFAULT_TOKEN_PATH = "/oauth/token"
88
87
  METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
89
88
  SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
90
- NAME = re.compile(r"^[a-z][a-z0-9_]*$")
91
89
  #: Vendor-named things — query parameters, path arguments, redacted fields — keep the
92
90
  #: vendor's casing (``startDate``, ``Limit``); only our own operation names are snake_case.
93
91
  FIELD = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
@@ -178,6 +176,29 @@ class Spec:
178
176
  fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
179
177
  return self.operations[name]
180
178
 
179
+ def check_entities(self, declared: Iterable[str]) -> None:
180
+ """Every operation that lands on an entity lands on one the world declares.
181
+
182
+ ``parse_spec`` reads connector.toml alone, so this is the step that binds the
183
+ connection to ``schema/world.json``; compile, check and the platform's describe
184
+ all call it, so a route onto a ghost entity is refused before it is served."""
185
+ known = set(declared)
186
+ for index, operation in enumerate(self.operations.values()):
187
+ landings = [(operation.entity, f"/operations/{index}/entity")] + [
188
+ (child.entity, f"/operations/{index}/ingest/explode/{position}/entity")
189
+ for position, child in enumerate(operation.explode)
190
+ ]
191
+ for entity, location in landings:
192
+ if entity is None or entity in known:
193
+ continue
194
+ fail(
195
+ "spec",
196
+ f"operation {operation.name} lands on entity {entity}, which schema/world.json "
197
+ f"does not declare (declared: {', '.join(sorted(known)) or 'none'}); add the entity with "
198
+ "`gateway worlds schema entity add`, or point the route at one that exists",
199
+ location,
200
+ )
201
+
181
202
 
182
203
  # ---------------------------------------------------------------- spec
183
204
 
@@ -300,10 +321,11 @@ def _positive(table: dict, key: str, location: str, default: int | float, *, zer
300
321
 
301
322
 
302
323
  def parse_spec(text: str) -> Spec:
303
- try:
304
- raw = tomllib.loads(text)
305
- except tomllib.TOMLDecodeError as error:
306
- fail("spec", f"connector.toml is not valid TOML: {error}", "/connector.toml")
324
+ return spec_from(parse_toml(text))
325
+
326
+
327
+ def spec_from(raw: dict) -> Spec:
328
+ """The ``Spec`` of a parsed connector.toml, for a caller that also reads its other tables."""
307
329
  connector = _table(raw.get("connector"), "/connector")
308
330
  slug = _string(connector, "slug", "/connector")
309
331
  if not SLUG.match(slug):
@@ -356,8 +378,7 @@ def parse_spec(text: str) -> Spec:
356
378
  location = f"/operations/{index}"
357
379
  table = _table(entry, location)
358
380
  name = _string(table, "name", location)
359
- if not NAME.match(name):
360
- fail("spec", "operation name must be snake_case", f"{location}/name")
381
+ operation_name(name, f"{location}/name")
361
382
  if name in operations:
362
383
  fail("spec", f"operation {name} is declared twice", f"{location}/name")
363
384
  methods = _verbs(table, location)
@@ -369,14 +390,20 @@ def parse_spec(text: str) -> Spec:
369
390
  path = _string(table, "path", location)
370
391
  if not path.startswith("/") or "?" in path or "#" in path:
371
392
  fail("spec", "path must start with / and carry no query", f"{location}/path")
393
+ placeholders = PLACEHOLDER.findall(path)
394
+ repeated = [p for p in placeholders if placeholders.count(p) > 1]
395
+ if repeated:
396
+ fail(
397
+ "spec",
398
+ f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
399
+ f"{location}/path",
400
+ argument=repeated[0],
401
+ )
372
402
  params = _names(table, "params", location, PARAM)
373
403
  entity = _string(table, "entity", location, required=False)
374
404
  results = table.get("results", "")
375
405
  if not isinstance(results, str):
376
406
  fail("spec", "results must be a dotted path string", f"{location}/results")
377
- for placeholder in PLACEHOLDER.findall(path):
378
- if placeholder in params:
379
- fail("spec", f"{placeholder} is both a path argument and a query parameter", location)
380
407
  drop, explode, carry_args = _ingest_projection(table.get("ingest", {}), f"{location}/ingest")
381
408
  if (drop or explode or carry_args) and not entity:
382
409
  fail("spec", "ingest projection needs an entity to land rows in", f"{location}/ingest")
@@ -457,11 +484,29 @@ def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[
457
484
  return drop, tuple(explode), carry_args
458
485
 
459
486
 
487
+ def bind_spec(spec: Spec, world_document: object) -> Spec:
488
+ """``spec`` checked against a parsed world.json (:meth:`Spec.check_entities`); a
489
+ document without a usable ``x-gateway.entities`` map declares nothing."""
490
+ annotation = world_document.get("x-gateway") if isinstance(world_document, dict) else None
491
+ entities = annotation.get("entities") if isinstance(annotation, dict) else None
492
+ spec.check_entities(entities if isinstance(entities, dict) else {})
493
+ return spec
494
+
495
+
460
496
  def load_spec(world: str | Path) -> Spec:
497
+ """The world's connection, bound to its schema when the tree has one: a route
498
+ onto an entity ``schema/world.json`` does not declare is refused here too, so
499
+ serve, open, `worlds test`, describe and ingest read the same rule compile and check do."""
461
500
  path = Path(world) / SPEC
462
501
  if not path.is_file():
463
502
  fail("spec", f"No {SPEC} in the world; describe the connection first", f"/{SPEC}")
464
- return parse_spec(path.read_text(encoding="utf-8"))
503
+ spec = parse_spec(path.read_text(encoding="utf-8"))
504
+ world_json = Path(world) / "schema/world.json"
505
+ if not world_json.is_file():
506
+ return spec
507
+ from .validation import read_json
508
+
509
+ return bind_spec(spec, read_json(world_json))
465
510
 
466
511
 
467
512
  # ---------------------------------------------------------------- requests
@@ -1,4 +1,41 @@
1
- """Custom connector world tools; edit freely, but keep writes on the store."""
1
+ """Custom connector world tools; keep writes on the store.
2
+
3
+ Declare each tool under x-gateway.tools in schema/world.json and in
4
+ [tools].serve of gateway-env.toml, then implement it here:
5
+
6
+ from gateway_env import store, tool
7
+
8
+ @tool(name="list_records")
9
+ def list_records(kind):
10
+ return [r for r in store.list("records") if r["kind"] == kind]
11
+
12
+ `store` (from gateway_env) is the world's rows. It offers exactly these methods;
13
+ `gateway worlds schema check .` refuses any other `store.<name>` at file:line, and a
14
+ hosted call into such a handler answers 400 {error: "handler", detail, handler}:
15
+
16
+ get(entity: str, key: dict) -> dict | None
17
+ One row of `entity` by its primary key, or None when no row has that key.
18
+ list(entity: str) -> list[dict]
19
+ Every row of `entity`, ordered by key.
20
+ list_where(entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]
21
+ A page of `entity` rows whose every `where` field holds one of the listed values, plus the total match count.
22
+ insert(entity: str, value: dict, *, source: dict | None = None) -> dict
23
+ Validate `value` against the entity's schema and add it; the key must be new.
24
+ update(entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict
25
+ Merge `patch` into the row at `key` and validate the result; returns the row.
26
+ delete(entity: str, key: dict, *, source: dict | None = None) -> None
27
+ Remove the row at `key`; a missing row is a contract error.
28
+ apply_batch(batch: dict) -> dict
29
+ Apply one ingest batch {id, target, mapping, coverage, mutations} atomically; an exact retry is a no-op.
30
+ lineage(entity: str, key: dict) -> list[dict]
31
+ The batches that wrote the row at `key`, oldest first, with each batch's source.
32
+ counts() -> dict[str, int]
33
+ Rows per entity, as {entity: count}.
34
+
35
+ A handler that raises AttributeError, TypeError or NameError is answered 400 with
36
+ that structured body; any other exception is answered 500 {error: "engine", ...}.
37
+ `schema init --connector custom --with-example-tool` writes the example in full.
38
+ """
2
39
 
3
40
  from gateway_env import store, tool
4
41
 
@@ -1,5 +1,7 @@
1
1
  # How the platform fills a world on this contract from a live Merge connection
2
- # (`gateway worlds create --template google-drive --mode live --connection <id>`).
2
+ # (`gateway worlds create <slug> --from-connection <id|google_drive>`, refreshed with
3
+ # `gateway worlds data pull <slug> --connection <id|google_drive>`; MCP `create_world` /
4
+ # `pull_world_data`).
3
5
  # Each [[pulls]] is one read tool of Merge's `google_drive` tool pack, called on
4
6
  # the connection's registered user; readers/merge.ingest.toml maps what comes back
5
7
  # onto the world's entities. Only read tools belong here.
@@ -0,0 +1,20 @@
1
+ """How a ``basic`` world's credential rides a request: as the Basic *user*,
2
+ with an empty password (``Basic b64("<key>:")``). The host and the local
3
+ server present it that way (``api.vendor_credential``), and the world
4
+ (``WorldApp.authorized``) and the host's gate (``vendor_shaped_token``) read
5
+ it back with :func:`basic_user`, so the two sides cannot drift."""
6
+
7
+ from __future__ import annotations
8
+
9
+ import base64
10
+
11
+
12
+ def basic_user(authorization: str) -> str | None:
13
+ """The user of a ``Basic`` authorization header, or None when it is not one."""
14
+ if not authorization.lower().startswith("basic "):
15
+ return None
16
+ try:
17
+ decoded = base64.b64decode(authorization[6:].strip().encode("ascii"), validate=True).decode("utf-8")
18
+ except (ValueError, UnicodeDecodeError):
19
+ return None
20
+ return decoded.split(":", 1)[0]