@withgateway/sdk 0.1.19 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -2
- package/dist/cli.js +8677 -1320
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +8683 -1327
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1005 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +969 -47
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +9 -2
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +9 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +738 -28
- package/dist/worlds/index.d.ts +738 -28
- package/dist/worlds/index.js +966 -51
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +960 -45
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +9 -2
- package/runtime/gatewaysdk/__init__.py +0 -0
- package/runtime/gatewaysdk/environment/__init__.py +108 -0
- package/runtime/gatewaysdk/environment/_bundle.py +281 -0
- package/runtime/gatewaysdk/environment/_harbor.py +276 -0
- package/runtime/gatewaysdk/environment/_materialize.py +97 -0
- package/runtime/gatewaysdk/environment/_tar.py +33 -0
- package/runtime/gatewaysdk/environment/_world.py +346 -0
- package/runtime/gatewaysdk/environment/core.py +527 -0
- package/runtime/gatewaysdk/environment/errors.py +58 -0
- package/runtime/gatewaysdk/environment/runtime.py +150 -0
- package/runtime/gatewaysdk/environment/schema/__init__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +225 -0
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +116 -0
- package/runtime/gatewaysdk/environment/schema/_slack_scenario.py +183 -0
- package/runtime/gatewaysdk/environment/schema/api.py +1628 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +755 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +615 -0
- package/runtime/gatewaysdk/environment/schema/conform.py +428 -0
- package/runtime/gatewaysdk/environment/schema/connector.py +777 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/handlers.py +17 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/parity.json +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/provenance.json +233 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/scope.toml +112 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/world.json +1380 -0
- package/runtime/gatewaysdk/environment/schema/connectors/base.json +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +18 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/world.json +123 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/handlers.py +81 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/parity.json +37 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/provenance.json +134 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/scope.toml +158 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +5869 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/handlers.py +23 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/world.json +209 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/handlers.py +18 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/world.json +286 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/handlers.py +34 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/parity.json +49 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +181 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +128 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/world.json +2218 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jira/handlers.py +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jira/world.json +389 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/handlers.py +38 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/parity.json +29 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/provenance.json +152 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/scope.toml +120 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/world.json +765 -0
- package/runtime/gatewaysdk/environment/schema/connectors/linear/handlers.py +20 -0
- package/runtime/gatewaysdk/environment/schema/connectors/linear/world.json +363 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/fidelity.json +45 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/handlers.py +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/provenance.json +851 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/scope.md +231 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/teams-types.json +1710 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/world.json +858 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/handlers.py +32 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/parity.json +50 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/provenance.json +137 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/scope.toml +78 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/world.json +16243 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/handlers.py +806 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/parity.json +46 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/provenance.json +143 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/scope.toml +174 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/world.json +4695 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/MODEL.md +236 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/capabilities.json +1039 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/handlers.py +762 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/parity.json +65 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/provenance.json +294 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/scope.toml +44 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/world.json +4993 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/handlers.py +75 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/parity.json +36 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/provenance.json +179 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/scope.toml +180 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/world.json +806 -0
- package/runtime/gatewaysdk/environment/schema/data.py +593 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +263 -0
- package/runtime/gatewaysdk/environment/schema/host.py +2407 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +275 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +1272 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +250 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +118 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +202 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +313 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +223 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +293 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +390 -0
- package/runtime/gatewaysdk/environment/schema/slack.py +757 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +479 -0
- package/runtime/gatewaysdk/environment/schema/store.py +904 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +525 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +322 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +577 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +78 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +500 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +336 -0
- package/skills/agents/connector-template-author.md +202 -0
- package/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- package/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- package/skills/connector-schema-authoring/SKILL.md +313 -0
- package/skills/world-data-ingestion/ROWS.md +223 -0
- package/skills/world-data-ingestion/SKILL.md +293 -0
- package/skills/worlds-getting-started/SKILL.md +390 -0
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
"""The deliberately bounded, offline Gateway JSON Schema dialect.
|
|
2
|
+
|
|
3
|
+
This is not a general JSON Schema validator. Schema documents are checked before
|
|
4
|
+
use; unsupported assertions fail rather than becoming unenforced promises.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import datetime as dt
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import re
|
|
14
|
+
import struct
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, NoReturn
|
|
17
|
+
|
|
18
|
+
DIALECT = "https://json-schema.org/draft/2020-12/schema"
|
|
19
|
+
LANGUAGE = "gateway-world/1"
|
|
20
|
+
SAFE_INTEGER = 2**53 - 1
|
|
21
|
+
DOC_KEYS = {
|
|
22
|
+
"title",
|
|
23
|
+
"description",
|
|
24
|
+
"$comment",
|
|
25
|
+
"examples",
|
|
26
|
+
"deprecated",
|
|
27
|
+
"readOnly",
|
|
28
|
+
"writeOnly",
|
|
29
|
+
"default",
|
|
30
|
+
}
|
|
31
|
+
TYPES = {"object", "array", "string", "integer", "number", "boolean", "null"}
|
|
32
|
+
KEYWORDS = DOC_KEYS | {
|
|
33
|
+
"$schema",
|
|
34
|
+
"$id",
|
|
35
|
+
"$defs",
|
|
36
|
+
"$ref",
|
|
37
|
+
"type",
|
|
38
|
+
"properties",
|
|
39
|
+
"required",
|
|
40
|
+
"additionalProperties",
|
|
41
|
+
"items",
|
|
42
|
+
"minItems",
|
|
43
|
+
"maxItems",
|
|
44
|
+
"uniqueItems",
|
|
45
|
+
"enum",
|
|
46
|
+
"const",
|
|
47
|
+
"minLength",
|
|
48
|
+
"maxLength",
|
|
49
|
+
"minimum",
|
|
50
|
+
"maximum",
|
|
51
|
+
"exclusiveMinimum",
|
|
52
|
+
"exclusiveMaximum",
|
|
53
|
+
"format",
|
|
54
|
+
"allOf",
|
|
55
|
+
"anyOf",
|
|
56
|
+
"oneOf",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ContractError(ValueError):
|
|
61
|
+
"""Payload-free structured diagnostics, safe to surface to authors."""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
code: str,
|
|
66
|
+
message: str,
|
|
67
|
+
*,
|
|
68
|
+
schema_path: str = "",
|
|
69
|
+
path: str = "",
|
|
70
|
+
**context: Any,
|
|
71
|
+
):
|
|
72
|
+
self.detail = {
|
|
73
|
+
"code": code,
|
|
74
|
+
"message": message,
|
|
75
|
+
"schema_path": schema_path,
|
|
76
|
+
"path": path,
|
|
77
|
+
**context,
|
|
78
|
+
}
|
|
79
|
+
super().__init__(f"{code} at {schema_path or path or '/'}: {message}")
|
|
80
|
+
|
|
81
|
+
def contextualize(self, **context: Any) -> "ContractError":
|
|
82
|
+
self.detail.update(context)
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def fail(code: str, message: str, location: str = "", **context: Any) -> NoReturn:
|
|
87
|
+
raise ContractError(code, message, schema_path=location, **context)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def pointer(part: Any) -> str:
|
|
91
|
+
return str(part).replace("~", "~0").replace("/", "~1")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def json_value(value: Any, path: str = "") -> None:
|
|
95
|
+
"""Require JSON-native values and the explicitly supported numeric range."""
|
|
96
|
+
if value is None or type(value) is bool:
|
|
97
|
+
return
|
|
98
|
+
if type(value) is str:
|
|
99
|
+
try:
|
|
100
|
+
value.encode("utf-8")
|
|
101
|
+
except UnicodeError:
|
|
102
|
+
fail("json", "Unpaired Unicode surrogate is unsupported", path)
|
|
103
|
+
return
|
|
104
|
+
if type(value) in (int, float):
|
|
105
|
+
if abs(value) > SAFE_INTEGER or not math.isfinite(value):
|
|
106
|
+
fail("numeric_range", "Numbers must be finite and within ±(2^53-1)", path)
|
|
107
|
+
return
|
|
108
|
+
if type(value) is list:
|
|
109
|
+
for index, item in enumerate(value):
|
|
110
|
+
json_value(item, f"{path}/{index}")
|
|
111
|
+
return
|
|
112
|
+
if type(value) is dict and all(type(k) is str for k in value):
|
|
113
|
+
for key, item in value.items():
|
|
114
|
+
json_value(key, path)
|
|
115
|
+
json_value(item, f"{path}/{pointer(key)}")
|
|
116
|
+
return
|
|
117
|
+
fail("json", "Expected JSON-native values (no coercion)", path)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def canonical(value: Any) -> str:
|
|
121
|
+
json_value(value)
|
|
122
|
+
return json.dumps(
|
|
123
|
+
value,
|
|
124
|
+
sort_keys=True,
|
|
125
|
+
ensure_ascii=False,
|
|
126
|
+
separators=(",", ":"),
|
|
127
|
+
allow_nan=False,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def encoded(value: Any) -> bytes:
|
|
132
|
+
return (canonical(value) + "\n").encode("utf-8")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def digest(value: bytes) -> str:
|
|
136
|
+
return hashlib.sha256(value).hexdigest()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def journal_paths(root: Path) -> list[Path]:
|
|
140
|
+
"""``data/initial.json`` then every refresh journal, in replay order."""
|
|
141
|
+
paths = [root / "data/initial.json"]
|
|
142
|
+
batches = root / "data/batches"
|
|
143
|
+
if batches.is_dir() and not batches.is_symlink():
|
|
144
|
+
paths.extend(sorted(p for p in batches.iterdir() if p.suffix == ".json"))
|
|
145
|
+
return [p for p in paths if p.is_file()]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def publication_digest(value: Any) -> str:
|
|
149
|
+
"""Cross-runtime canonical hash used by the Node publication gate."""
|
|
150
|
+
json_value(value)
|
|
151
|
+
|
|
152
|
+
def encode(item: Any) -> bytes:
|
|
153
|
+
if item is None:
|
|
154
|
+
return b"n"
|
|
155
|
+
if type(item) is bool:
|
|
156
|
+
return b"t" if item else b"f"
|
|
157
|
+
if type(item) in (int, float):
|
|
158
|
+
number = 0.0 if item == 0 else float(item)
|
|
159
|
+
return b"d" + struct.pack(">d", number).hex().encode("ascii")
|
|
160
|
+
if type(item) is str:
|
|
161
|
+
body = item.encode("utf-8")
|
|
162
|
+
return b"s" + str(len(body)).encode("ascii") + b":" + body
|
|
163
|
+
if type(item) is list:
|
|
164
|
+
return b"a" + str(len(item)).encode("ascii") + b":" + b"".join(encode(child) for child in item)
|
|
165
|
+
if type(item) is dict:
|
|
166
|
+
keys = sorted(item, key=lambda key: key.encode("utf-8"))
|
|
167
|
+
return (
|
|
168
|
+
b"o" + str(len(keys)).encode("ascii") + b":" + b"".join(encode(key) + encode(item[key]) for key in keys)
|
|
169
|
+
)
|
|
170
|
+
fail("json", "Expected JSON-native values (no coercion)")
|
|
171
|
+
|
|
172
|
+
return digest(encode(value))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def parse_json(content: str | bytes, location: str = "") -> Any:
|
|
176
|
+
"""Decode contract JSON without silently collapsing duplicate properties."""
|
|
177
|
+
|
|
178
|
+
def pairs(items):
|
|
179
|
+
result = {}
|
|
180
|
+
for key, value in items:
|
|
181
|
+
if key in result:
|
|
182
|
+
fail("duplicate_key", f"Duplicate JSON member {key}", location)
|
|
183
|
+
result[key] = value
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
result = json.loads(content, object_pairs_hook=pairs)
|
|
187
|
+
json_value(result, location)
|
|
188
|
+
return result
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def read_json(path: Path) -> Any:
|
|
192
|
+
try:
|
|
193
|
+
return parse_json(path.read_text(encoding="utf-8"), path.name)
|
|
194
|
+
except (OSError, ValueError, UnicodeError) as error:
|
|
195
|
+
if isinstance(error, ContractError):
|
|
196
|
+
raise
|
|
197
|
+
fail("document", "Cannot read valid UTF-8 JSON", path.name)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def members(value: Any, allowed: set[str], required: set[str], location: str) -> None:
|
|
201
|
+
if type(value) is not dict:
|
|
202
|
+
fail("schema", "Expected an object", location)
|
|
203
|
+
unknown = set(value) - allowed
|
|
204
|
+
missing = required - set(value)
|
|
205
|
+
if unknown:
|
|
206
|
+
fail(
|
|
207
|
+
"unsupported",
|
|
208
|
+
f"Unsupported member {sorted(unknown)[0]}",
|
|
209
|
+
f"{location}/{pointer(sorted(unknown)[0])}",
|
|
210
|
+
)
|
|
211
|
+
if missing:
|
|
212
|
+
fail("schema", f"Missing member {sorted(missing)[0]}", location)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def nonempty(value: Any, location: str) -> None:
|
|
216
|
+
if type(value) is not str or not value.strip() or len(value) > 256:
|
|
217
|
+
fail("schema", "Expected a nonempty string of at most 256 characters", location)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def equal(left: Any, right: Any) -> bool:
|
|
221
|
+
"""JSON equality: bool is distinct from number; 1 and 1.0 are equal."""
|
|
222
|
+
if type(left) in (int, float) and type(right) in (int, float):
|
|
223
|
+
return left == right
|
|
224
|
+
if type(left) is not type(right):
|
|
225
|
+
return False
|
|
226
|
+
if isinstance(left, dict):
|
|
227
|
+
return left.keys() == right.keys() and all(equal(left[k], right[k]) for k in left)
|
|
228
|
+
if isinstance(left, list):
|
|
229
|
+
return len(left) == len(right) and all(equal(a, b) for a, b in zip(left, right))
|
|
230
|
+
return left == right
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def validate(schema: dict, value: Any, *, location: str = "", path: str = "") -> None:
|
|
234
|
+
json_value(value, path)
|
|
235
|
+
|
|
236
|
+
def error(code: str, message: str, keyword: str = "") -> NoReturn:
|
|
237
|
+
fail(code, message, f"{location}/{keyword}" if keyword else location, path=path)
|
|
238
|
+
|
|
239
|
+
expected = schema.get("type")
|
|
240
|
+
types = expected if isinstance(expected, list) else [expected] if expected else []
|
|
241
|
+
actual = {
|
|
242
|
+
type(None): "null",
|
|
243
|
+
bool: "boolean",
|
|
244
|
+
str: "string",
|
|
245
|
+
dict: "object",
|
|
246
|
+
list: "array",
|
|
247
|
+
int: "integer",
|
|
248
|
+
float: "number",
|
|
249
|
+
}[type(value)]
|
|
250
|
+
matches = (
|
|
251
|
+
actual in types
|
|
252
|
+
or (actual == "integer" and "number" in types)
|
|
253
|
+
or (actual == "number" and "integer" in types and value.is_integer())
|
|
254
|
+
)
|
|
255
|
+
if types and not matches:
|
|
256
|
+
error("type", f"Expected {' or '.join(types)}", "type")
|
|
257
|
+
if "enum" in schema and not any(equal(value, option) for option in schema["enum"]):
|
|
258
|
+
error("enum", "Value is outside the declared enum", "enum")
|
|
259
|
+
if "const" in schema and not equal(value, schema["const"]):
|
|
260
|
+
error("const", "Value differs from the declared constant", "const")
|
|
261
|
+
for keyword in ("allOf", "anyOf", "oneOf"):
|
|
262
|
+
if keyword not in schema:
|
|
263
|
+
continue
|
|
264
|
+
matches_count = 0
|
|
265
|
+
for index, branch in enumerate(schema[keyword]):
|
|
266
|
+
try:
|
|
267
|
+
validate(branch, value, location=f"{location}/{keyword}/{index}", path=path)
|
|
268
|
+
matches_count += 1
|
|
269
|
+
except ContractError:
|
|
270
|
+
pass
|
|
271
|
+
expected_count = len(schema[keyword]) if keyword == "allOf" else 1
|
|
272
|
+
if (keyword == "anyOf" and not matches_count) or (keyword != "anyOf" and matches_count != expected_count):
|
|
273
|
+
error(keyword, "Value does not satisfy the composition", keyword)
|
|
274
|
+
if type(value) is dict:
|
|
275
|
+
properties = schema.get("properties", {})
|
|
276
|
+
for name in schema.get("required", []):
|
|
277
|
+
if name not in value:
|
|
278
|
+
error("required", f"Missing required property {name}", "required")
|
|
279
|
+
for name, item in value.items():
|
|
280
|
+
if name in properties:
|
|
281
|
+
validate(
|
|
282
|
+
properties[name],
|
|
283
|
+
item,
|
|
284
|
+
location=f"{location}/properties/{pointer(name)}",
|
|
285
|
+
path=f"{path}/{pointer(name)}",
|
|
286
|
+
)
|
|
287
|
+
elif schema.get("additionalProperties") is False:
|
|
288
|
+
error(
|
|
289
|
+
"additionalProperties",
|
|
290
|
+
f"Unknown property {name}",
|
|
291
|
+
"additionalProperties",
|
|
292
|
+
)
|
|
293
|
+
if type(value) is list:
|
|
294
|
+
for keyword, invalid in (
|
|
295
|
+
("minItems", len(value) < schema.get("minItems", 0)),
|
|
296
|
+
("maxItems", len(value) > schema.get("maxItems", math.inf)),
|
|
297
|
+
):
|
|
298
|
+
if invalid:
|
|
299
|
+
error(keyword, "Array length is outside the declared bound", keyword)
|
|
300
|
+
if schema.get("uniqueItems") and any(
|
|
301
|
+
equal(item, previous) for i, item in enumerate(value) for previous in value[:i]
|
|
302
|
+
):
|
|
303
|
+
error("uniqueItems", "Array items must be unique", "uniqueItems")
|
|
304
|
+
if "items" in schema:
|
|
305
|
+
for index, item in enumerate(value):
|
|
306
|
+
validate(
|
|
307
|
+
schema["items"],
|
|
308
|
+
item,
|
|
309
|
+
location=f"{location}/items",
|
|
310
|
+
path=f"{path}/{index}",
|
|
311
|
+
)
|
|
312
|
+
if type(value) is str:
|
|
313
|
+
if len(value) < schema.get("minLength", 0) or len(value) > schema.get("maxLength", math.inf):
|
|
314
|
+
error("length", "String length is outside the declared bound")
|
|
315
|
+
format_name = schema.get("format")
|
|
316
|
+
if format_name in {"date", "time", "date-time", "vendor-date-time"}:
|
|
317
|
+
try:
|
|
318
|
+
offset = r"(?:[Zz]|[+-](?:[01]\d|2[0-3]):[0-5]\d)"
|
|
319
|
+
clock = r"(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?"
|
|
320
|
+
if format_name == "date":
|
|
321
|
+
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
322
|
+
raise ValueError
|
|
323
|
+
dt.date.fromisoformat(value)
|
|
324
|
+
elif format_name == "time":
|
|
325
|
+
if not re.fullmatch(rf"{clock}{offset}", value):
|
|
326
|
+
raise ValueError
|
|
327
|
+
dt.time.fromisoformat(value.upper().replace("Z", "+00:00"))
|
|
328
|
+
elif format_name == "date-time":
|
|
329
|
+
if not re.fullmatch(rf"\d{{4}}-\d{{2}}-\d{{2}}[Tt]{clock}{offset}", value):
|
|
330
|
+
raise ValueError
|
|
331
|
+
stamp = dt.datetime.fromisoformat(value.upper().replace("Z", "+00:00"))
|
|
332
|
+
if stamp.utcoffset() is None:
|
|
333
|
+
raise ValueError
|
|
334
|
+
elif not re.fullmatch(rf"\d{{4}}-\d{{2}}-\d{{2}}[ Tt]{clock}(?:{offset})?", value):
|
|
335
|
+
raise ValueError
|
|
336
|
+
else:
|
|
337
|
+
dt.datetime.fromisoformat(value.upper().replace("Z", "+00:00"))
|
|
338
|
+
except ValueError:
|
|
339
|
+
error("format", f"Expected a valid {format_name} value", "format")
|
|
340
|
+
if type(value) in (int, float):
|
|
341
|
+
for keyword, comparison in (
|
|
342
|
+
("minimum", lambda b: value < b),
|
|
343
|
+
("maximum", lambda b: value > b),
|
|
344
|
+
("exclusiveMinimum", lambda b: value <= b),
|
|
345
|
+
("exclusiveMaximum", lambda b: value >= b),
|
|
346
|
+
):
|
|
347
|
+
if keyword in schema and comparison(schema[keyword]):
|
|
348
|
+
error(keyword, "Number is outside the declared bound", keyword)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
class Resolver:
|
|
352
|
+
"""Resolve only relative files under schema/ and JSON pointers into $defs."""
|
|
353
|
+
|
|
354
|
+
def __init__(self, bundle: Path):
|
|
355
|
+
self.root = (bundle / "schema").resolve()
|
|
356
|
+
self.documents: dict[str, dict] = {}
|
|
357
|
+
self.identities: dict[str, str] = {}
|
|
358
|
+
self.checked: set[str] = set()
|
|
359
|
+
self.checking: set[str] = set()
|
|
360
|
+
|
|
361
|
+
def load(self, name: str) -> dict:
|
|
362
|
+
path = self.root / name
|
|
363
|
+
if path.resolve().parent != self.root or path.is_symlink() or not name.endswith(".json") or ":" in name:
|
|
364
|
+
fail(
|
|
365
|
+
"reference",
|
|
366
|
+
"References must be sibling .json files inside schema/",
|
|
367
|
+
name,
|
|
368
|
+
)
|
|
369
|
+
if name not in self.documents:
|
|
370
|
+
document = read_json(path)
|
|
371
|
+
if type(document) is not dict or document.get("$schema") != DIALECT:
|
|
372
|
+
fail("schema", f"Declare $schema = {DIALECT}", name)
|
|
373
|
+
nonempty(document.get("$id"), f"{name}/$id")
|
|
374
|
+
identity = document["$id"]
|
|
375
|
+
if (
|
|
376
|
+
not re.match(r"^[a-z][a-z0-9+.-]*:", identity)
|
|
377
|
+
or "#" in identity
|
|
378
|
+
or re.search(r"[\s\x00-\x1f\x7f-\uffff]|%(?![0-9a-fA-F]{2})", identity)
|
|
379
|
+
):
|
|
380
|
+
fail(
|
|
381
|
+
"identity",
|
|
382
|
+
"$id must be an absolute, fragment-free URI (never fetched)",
|
|
383
|
+
f"{name}/$id",
|
|
384
|
+
)
|
|
385
|
+
if identity in self.identities and self.identities[identity] != name:
|
|
386
|
+
fail("identity", "Duplicate schema $id", f"{name}/$id")
|
|
387
|
+
self.identities[identity] = name
|
|
388
|
+
self.documents[name] = document
|
|
389
|
+
return self.documents[name]
|
|
390
|
+
|
|
391
|
+
def reference(self, ref: str, name: str, location: str) -> tuple[dict, str, str]:
|
|
392
|
+
if type(ref) is not str or ref.count("#") != 1:
|
|
393
|
+
fail("reference", "Use a local #/$defs/name reference", location)
|
|
394
|
+
filename, fragment = ref.split("#")
|
|
395
|
+
target_name = filename or name
|
|
396
|
+
if filename and not re.fullmatch(r"[A-Za-z0-9_.-]+\.json", filename):
|
|
397
|
+
fail("reference", "Remote and escaping references are unsupported", location)
|
|
398
|
+
if not fragment.startswith("/$defs/"):
|
|
399
|
+
fail("reference", "References must point into $defs", location)
|
|
400
|
+
current = self.load(target_name)
|
|
401
|
+
for part in fragment[1:].split("/"):
|
|
402
|
+
if re.search(r"~(?![01])", part):
|
|
403
|
+
fail("reference", "Invalid JSON pointer escape", location)
|
|
404
|
+
part = part.replace("~1", "/").replace("~0", "~")
|
|
405
|
+
if not isinstance(current, dict) or part not in current:
|
|
406
|
+
fail("reference", f"Unresolved reference {ref}", location)
|
|
407
|
+
current = current[part]
|
|
408
|
+
return current, target_name, f"{target_name}#{fragment}"
|
|
409
|
+
|
|
410
|
+
def resolve(
|
|
411
|
+
self,
|
|
412
|
+
schema: Any,
|
|
413
|
+
name: str,
|
|
414
|
+
location: str,
|
|
415
|
+
stack: tuple[str, ...] = (),
|
|
416
|
+
*,
|
|
417
|
+
root: bool = False,
|
|
418
|
+
) -> dict:
|
|
419
|
+
if len(stack) > 80:
|
|
420
|
+
fail("reference", "Reference nesting exceeds 80", location)
|
|
421
|
+
if type(schema) is not dict:
|
|
422
|
+
fail(
|
|
423
|
+
"unsupported",
|
|
424
|
+
"Boolean schemas are unsupported; use an explicit type",
|
|
425
|
+
location,
|
|
426
|
+
)
|
|
427
|
+
allowed = KEYWORDS | ({"x-gateway"} if root and name == "world.json" else set())
|
|
428
|
+
members(schema, allowed, set(), location)
|
|
429
|
+
if not root and ("$id" in schema or "$schema" in schema or "x-gateway" in schema):
|
|
430
|
+
fail(
|
|
431
|
+
"unsupported",
|
|
432
|
+
"Identity/dialect/annotations belong at document root",
|
|
433
|
+
location,
|
|
434
|
+
)
|
|
435
|
+
for key in ("title", "description", "$comment"):
|
|
436
|
+
if key in schema and type(schema[key]) is not str:
|
|
437
|
+
fail("schema", f"{key} must be a string", location)
|
|
438
|
+
for key in (
|
|
439
|
+
"deprecated",
|
|
440
|
+
"readOnly",
|
|
441
|
+
"writeOnly",
|
|
442
|
+
"uniqueItems",
|
|
443
|
+
"additionalProperties",
|
|
444
|
+
):
|
|
445
|
+
if key in schema and type(schema[key]) is not bool:
|
|
446
|
+
fail("schema", f"{key} must be boolean", f"{location}/{key}")
|
|
447
|
+
if "examples" in schema and type(schema["examples"]) is not list:
|
|
448
|
+
fail("schema", "examples must be an array", location)
|
|
449
|
+
result = {k: v for k, v in schema.items() if k not in {"$ref", "$defs", "$schema", "$id", "x-gateway"}}
|
|
450
|
+
if "$ref" in schema:
|
|
451
|
+
if set(schema) - DOC_KEYS - {"$ref"}:
|
|
452
|
+
fail(
|
|
453
|
+
"unsupported",
|
|
454
|
+
"$ref siblings may only be documentation annotations; use allOf for assertions",
|
|
455
|
+
location,
|
|
456
|
+
)
|
|
457
|
+
target, target_name, target_location = self.reference(schema["$ref"], name, location)
|
|
458
|
+
if target_location in stack:
|
|
459
|
+
fail(
|
|
460
|
+
"recursive_reference",
|
|
461
|
+
"Recursive payload schemas are unsupported (entity relationships may cycle)",
|
|
462
|
+
location,
|
|
463
|
+
)
|
|
464
|
+
result = {
|
|
465
|
+
**self.resolve(target, target_name, target_location, (*stack, target_location)),
|
|
466
|
+
**result,
|
|
467
|
+
}
|
|
468
|
+
for key in ("$defs", "properties"):
|
|
469
|
+
if key not in schema:
|
|
470
|
+
continue
|
|
471
|
+
if type(schema[key]) is not dict:
|
|
472
|
+
fail("schema", f"{key} must be an object", f"{location}/{key}")
|
|
473
|
+
children = {
|
|
474
|
+
k: self.resolve(v, name, f"{location}/{key}/{pointer(k)}", stack)
|
|
475
|
+
for k, v in sorted(schema[key].items())
|
|
476
|
+
}
|
|
477
|
+
if key == "properties":
|
|
478
|
+
result[key] = children
|
|
479
|
+
if "items" in schema:
|
|
480
|
+
result["items"] = self.resolve(schema["items"], name, f"{location}/items", stack)
|
|
481
|
+
for key in ("allOf", "anyOf", "oneOf"):
|
|
482
|
+
if key in schema:
|
|
483
|
+
if not isinstance(schema[key], list) or not schema[key]:
|
|
484
|
+
fail("schema", f"{key} needs a nonempty schema array", location)
|
|
485
|
+
result[key] = [self.resolve(s, name, f"{location}/{key}/{i}", stack) for i, s in enumerate(schema[key])]
|
|
486
|
+
types = result.get("type", [])
|
|
487
|
+
types = [types] if isinstance(types, str) else types
|
|
488
|
+
if (
|
|
489
|
+
not isinstance(types, list)
|
|
490
|
+
or any(type(t) is not str or t not in TYPES for t in types)
|
|
491
|
+
or len(types) != len(set(types))
|
|
492
|
+
):
|
|
493
|
+
fail("schema", "Invalid type declaration", location)
|
|
494
|
+
if "type" in result and not types:
|
|
495
|
+
fail("schema", "type arrays must not be empty", location)
|
|
496
|
+
if len(types) > 1 and (len(types) != 2 or "null" not in types):
|
|
497
|
+
fail(
|
|
498
|
+
"unsupported",
|
|
499
|
+
"Type unions support one type plus null; use anyOf otherwise",
|
|
500
|
+
location,
|
|
501
|
+
)
|
|
502
|
+
if not types and not any(k in result for k in ("allOf", "anyOf", "oneOf")) and not root:
|
|
503
|
+
fail("schema", "Declare an explicit type or composition", location)
|
|
504
|
+
if isinstance(result.get("type"), list):
|
|
505
|
+
result["type"] = sorted(types)
|
|
506
|
+
applicability = {
|
|
507
|
+
"properties": {"object"},
|
|
508
|
+
"required": {"object"},
|
|
509
|
+
"additionalProperties": {"object"},
|
|
510
|
+
"items": {"array"},
|
|
511
|
+
"minItems": {"array"},
|
|
512
|
+
"maxItems": {"array"},
|
|
513
|
+
"uniqueItems": {"array"},
|
|
514
|
+
"minLength": {"string"},
|
|
515
|
+
"maxLength": {"string"},
|
|
516
|
+
"format": {"string"},
|
|
517
|
+
"minimum": {"integer", "number"},
|
|
518
|
+
"maximum": {"integer", "number"},
|
|
519
|
+
"exclusiveMinimum": {"integer", "number"},
|
|
520
|
+
"exclusiveMaximum": {"integer", "number"},
|
|
521
|
+
}
|
|
522
|
+
for key, applicable in applicability.items():
|
|
523
|
+
if key in result and not set(types) & applicable:
|
|
524
|
+
fail(
|
|
525
|
+
"schema",
|
|
526
|
+
f"{key} requires an applicable explicit type",
|
|
527
|
+
f"{location}/{key}",
|
|
528
|
+
)
|
|
529
|
+
if "array" in types and "items" not in result:
|
|
530
|
+
fail("schema", "Array schemas require items", location)
|
|
531
|
+
if "required" in result:
|
|
532
|
+
required = result["required"]
|
|
533
|
+
if (
|
|
534
|
+
not isinstance(required, list)
|
|
535
|
+
or any(type(k) is not str for k in required)
|
|
536
|
+
or len(set(required)) != len(required)
|
|
537
|
+
):
|
|
538
|
+
fail("schema", "required must contain distinct property names", location)
|
|
539
|
+
if set(required) - set(result.get("properties", {})):
|
|
540
|
+
fail("schema", "required names must be declared properties", location)
|
|
541
|
+
result["required"] = sorted(required)
|
|
542
|
+
for key in ("minItems", "maxItems", "minLength", "maxLength"):
|
|
543
|
+
if key in result and (type(result[key]) is not int or result[key] < 0):
|
|
544
|
+
fail("schema", f"{key} must be a nonnegative integer", location)
|
|
545
|
+
for key in ("minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"):
|
|
546
|
+
if key in result and type(result[key]) not in (int, float):
|
|
547
|
+
fail("schema", f"{key} must be numeric", location)
|
|
548
|
+
if "format" in result and result["format"] not in {
|
|
549
|
+
"date",
|
|
550
|
+
"time",
|
|
551
|
+
"date-time",
|
|
552
|
+
"vendor-date-time",
|
|
553
|
+
}:
|
|
554
|
+
fail(
|
|
555
|
+
"unsupported",
|
|
556
|
+
"Unsupported string format profile",
|
|
557
|
+
f"{location}/format",
|
|
558
|
+
)
|
|
559
|
+
if "enum" in result:
|
|
560
|
+
values = result["enum"]
|
|
561
|
+
if (
|
|
562
|
+
not isinstance(values, list)
|
|
563
|
+
or not values
|
|
564
|
+
or any(equal(v, p) for i, v in enumerate(values) for p in values[:i])
|
|
565
|
+
):
|
|
566
|
+
fail("schema", "enum must contain distinct values", location)
|
|
567
|
+
if "default" in result:
|
|
568
|
+
validate(result, result["default"], location=f"{location}/default")
|
|
569
|
+
return result
|
|
570
|
+
|
|
571
|
+
def check_documents(self) -> None:
|
|
572
|
+
# Resolving references can discover more files. Check all definitions,
|
|
573
|
+
# including unused ones, so unsupported assertions never hide there.
|
|
574
|
+
while set(self.documents) - self.checked:
|
|
575
|
+
name = sorted(set(self.documents) - self.checked)[0]
|
|
576
|
+
self.resolve(self.documents[name], name, f"{name}#", root=True)
|
|
577
|
+
self.checked.add(name)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""``python -m gatewaysdk.environment.schema.worker``: one version's worker.
|
|
2
|
+
|
|
3
|
+
Started by the supervisor's :class:`workers.WorkerPool`, never by hand. It caps
|
|
4
|
+
and seals itself, then serves ONE version over the listening unix socket the
|
|
5
|
+
supervisor handed it (``--socket-fd``), answering the supervisor's pid alone,
|
|
6
|
+
running that version's ``handlers.py`` in this process and nowhere else. See
|
|
7
|
+
:mod:`workers` for what the boundary is and is not.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import os
|
|
14
|
+
import socket
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from .host import WorldHost, make_host_handler
|
|
19
|
+
from .workers import (
|
|
20
|
+
DEFAULT_WORKER_MEMORY_MB,
|
|
21
|
+
WORKER_LEASE_S,
|
|
22
|
+
UnixHTTPServer,
|
|
23
|
+
apply_limits,
|
|
24
|
+
seal_process,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def main(argv: list[str] | None = None) -> int:
|
|
29
|
+
parser = argparse.ArgumentParser(prog="gateway-world-worker")
|
|
30
|
+
parser.add_argument("--cache-root", required=True)
|
|
31
|
+
parser.add_argument("--version", required=True)
|
|
32
|
+
where = parser.add_mutually_exclusive_group(required=True)
|
|
33
|
+
where.add_argument("--socket-fd", type=int, help="listening unix socket inherited from the supervisor")
|
|
34
|
+
where.add_argument("--socket", help="path to bind (a hand-run worker; answers any peer)")
|
|
35
|
+
args = parser.parse_args(argv)
|
|
36
|
+
|
|
37
|
+
secret = os.environ.get("WORLDS_WORKER_SECRET") or ""
|
|
38
|
+
if not secret:
|
|
39
|
+
raise SystemExit("world worker: WORLDS_WORKER_SECRET is unset; a worker only answers its supervisor")
|
|
40
|
+
supervisor_pid: int | None = None
|
|
41
|
+
if args.socket_fd is not None:
|
|
42
|
+
supervisor_pid = int(os.environ.get("WORLDS_SUPERVISOR_PID") or os.getppid())
|
|
43
|
+
listening: socket.socket | str = socket.socket(fileno=args.socket_fd)
|
|
44
|
+
else:
|
|
45
|
+
listening = args.socket
|
|
46
|
+
seal_process()
|
|
47
|
+
limits = apply_limits(memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB))
|
|
48
|
+
|
|
49
|
+
cache_root = Path(args.cache_root)
|
|
50
|
+
world_host = WorldHost(
|
|
51
|
+
cache_root / "workers" / args.version,
|
|
52
|
+
versions_root=cache_root / "versions",
|
|
53
|
+
lease_s=WORKER_LEASE_S,
|
|
54
|
+
run_handlers=True,
|
|
55
|
+
pinned_version=args.version,
|
|
56
|
+
limits=limits,
|
|
57
|
+
)
|
|
58
|
+
handler = make_host_handler(
|
|
59
|
+
world_host,
|
|
60
|
+
secret=secret,
|
|
61
|
+
worlds_domain=os.environ.get("WORLDS_INGRESS_DOMAIN", "worlds.local"),
|
|
62
|
+
)
|
|
63
|
+
server = UnixHTTPServer(listening, handler, supervisor_pid=supervisor_pid)
|
|
64
|
+
where_bound = args.socket if args.socket else f"fd {args.socket_fd} for pid {supervisor_pid}"
|
|
65
|
+
print(
|
|
66
|
+
f"world worker for version {args.version} listening on {where_bound} (pid {os.getpid()})",
|
|
67
|
+
file=sys.stderr,
|
|
68
|
+
flush=True,
|
|
69
|
+
)
|
|
70
|
+
try:
|
|
71
|
+
server.serve_forever()
|
|
72
|
+
finally:
|
|
73
|
+
server.server_close()
|
|
74
|
+
return 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
raise SystemExit(main())
|