@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
|
@@ -23,6 +23,8 @@ from dataclasses import dataclass
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Any
|
|
25
25
|
|
|
26
|
+
from .validation import local_key
|
|
27
|
+
|
|
26
28
|
INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
|
|
27
29
|
NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
|
|
28
30
|
BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
|
|
@@ -33,19 +35,23 @@ _MISSING = object()
|
|
|
33
35
|
|
|
34
36
|
@dataclass(frozen=True)
|
|
35
37
|
class FieldType:
|
|
36
|
-
"""One field as ``gateway worlds describe`` reports it: ``type`` is the JSON Schema
|
|
37
|
-
|
|
38
|
-
``enum`` the values
|
|
38
|
+
"""One field as ``gateway worlds describe`` reports it: ``type`` is the JSON Schema type
|
|
39
|
+
words the field admits, through references and unions, joined by ``" | "``, or None when the
|
|
40
|
+
schema says nothing; ``enum`` the values it allows, when it lists every one; ``format`` the
|
|
41
|
+
schema's ``format`` (``date-time``, ``email``, ...), when it names one."""
|
|
39
42
|
|
|
40
43
|
name: str
|
|
41
44
|
type: str | None
|
|
42
45
|
required: bool
|
|
43
46
|
enum: tuple | None = None
|
|
47
|
+
format: str | None = None
|
|
44
48
|
|
|
45
49
|
def describe(self) -> dict:
|
|
46
50
|
described: dict = {"name": self.name, "type": self.type, "required": self.required}
|
|
47
51
|
if self.enum is not None:
|
|
48
52
|
described["enum"] = list(self.enum)
|
|
53
|
+
if self.format is not None:
|
|
54
|
+
described["format"] = self.format
|
|
49
55
|
return described
|
|
50
56
|
|
|
51
57
|
|
|
@@ -160,87 +166,147 @@ def casters_of(entities: dict[str, list[FieldType]]) -> EntityCasters:
|
|
|
160
166
|
return by_name.get
|
|
161
167
|
|
|
162
168
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
169
|
+
Resolve = Callable[[Any, frozenset], tuple[dict, frozenset]]
|
|
170
|
+
UNFOLLOWED = "$ref"
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def unresolved(schema: Any, seen: frozenset) -> tuple[dict, frozenset]:
|
|
174
|
+
return (schema if isinstance(schema, dict) else {}), seen
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class LocalDefinitions:
|
|
178
|
+
"""Follows ``#/$defs/<key>`` references into ``defs``; ``seen`` carries the keys on the path,
|
|
179
|
+
so a cycle through compositions ends."""
|
|
180
|
+
|
|
181
|
+
def __init__(self, defs: Any):
|
|
182
|
+
self.defs = defs if isinstance(defs, dict) else {}
|
|
183
|
+
|
|
184
|
+
def __call__(self, schema: Any, seen: frozenset = frozenset()) -> tuple[dict, frozenset]:
|
|
185
|
+
while isinstance(schema, dict):
|
|
186
|
+
key = local_key(schema.get("$ref"))
|
|
187
|
+
if key is None or key in seen or not isinstance(self.defs.get(key), dict):
|
|
188
|
+
break
|
|
189
|
+
seen = seen | {key}
|
|
190
|
+
schema = self.defs[key]
|
|
191
|
+
return unresolved(schema, seen)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def schema_types(schema: Any, resolve: Resolve = unresolved, seen: frozenset = frozenset()) -> tuple[str, ...]:
|
|
195
|
+
"""The JSON type words a schema admits, in the order it declares them, through references
|
|
196
|
+
and ``anyOf`` / ``oneOf`` / ``allOf``; :data:`UNFOLLOWED` for a reference ``resolve`` cannot follow
|
|
197
|
+
that carries no ``type`` of its own (a shared compile keeps the target's ``type`` beside its ``$ref``)."""
|
|
198
|
+
node, seen = resolve(schema, seen)
|
|
199
|
+
if isinstance(node.get("$ref"), str) and local_key(node["$ref"]) not in seen and "type" not in node:
|
|
200
|
+
return (UNFOLLOWED,)
|
|
201
|
+
kind = node.get("type")
|
|
202
|
+
found = [kind] if isinstance(kind, str) else [part for part in kind or [] if isinstance(part, str)]
|
|
203
|
+
for key in ("anyOf", "oneOf", "allOf"):
|
|
204
|
+
for branch in node.get(key) or []:
|
|
205
|
+
found += schema_types(branch, resolve, seen)
|
|
206
|
+
return tuple(dict.fromkeys(found))
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def allowed_values(
|
|
210
|
+
schema: Any, resolve: Resolve = unresolved, seen: frozenset = frozenset()
|
|
211
|
+
) -> tuple[list, list[dict]] | None:
|
|
212
|
+
"""The values a schema allows when it lists every one (``enum``, ``const``, a ``oneOf`` /
|
|
213
|
+
``anyOf`` of those with a ``null`` branch allowing None, or the values every listing ``allOf``
|
|
214
|
+
branch shares), with each labelled ``const``'s ``title``; None when it also admits values it
|
|
215
|
+
does not list."""
|
|
216
|
+
node, seen = resolve(schema, seen)
|
|
217
|
+
if isinstance(node.get("enum"), list):
|
|
218
|
+
return list(node["enum"]), []
|
|
219
|
+
if "const" in node:
|
|
220
|
+
label = node.get("title")
|
|
221
|
+
return [node["const"]], ([{"value": node["const"], "label": label}] if isinstance(label, str) else [])
|
|
222
|
+
if node.get("allOf") and not (node.get("oneOf") or node.get("anyOf")):
|
|
223
|
+
return _shared_values(node["allOf"], resolve, seen)
|
|
224
|
+
branches = node.get("oneOf") or node.get("anyOf")
|
|
225
|
+
if not isinstance(branches, list) or not branches:
|
|
226
|
+
return None
|
|
227
|
+
values: list = []
|
|
228
|
+
labels: list[dict] = []
|
|
229
|
+
for branch in branches:
|
|
230
|
+
if schema_types(branch, resolve, seen) == ("null",):
|
|
231
|
+
values.append(None)
|
|
232
|
+
continue
|
|
233
|
+
allowed = allowed_values(branch, resolve, seen)
|
|
234
|
+
if allowed is None:
|
|
235
|
+
return None
|
|
236
|
+
values += allowed[0]
|
|
237
|
+
labels += allowed[1]
|
|
238
|
+
return (values, labels) if [value for value in values if value is not None] else None
|
|
172
239
|
|
|
173
240
|
|
|
174
|
-
def
|
|
175
|
-
if
|
|
241
|
+
def _shared_values(branches: list, resolve: Resolve, seen: frozenset) -> tuple[list, list[dict]] | None:
|
|
242
|
+
listed = [allowed for branch in branches if (allowed := allowed_values(branch, resolve, seen)) is not None]
|
|
243
|
+
if not listed:
|
|
176
244
|
return None
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
245
|
+
values = [value for value in listed[0][0] if all(value in other[0] for other in listed[1:])]
|
|
246
|
+
labels = [label for _, branch_labels in listed for label in branch_labels if label["value"] in values]
|
|
247
|
+
labels = [label for index, label in enumerate(labels) if label not in labels[:index]]
|
|
248
|
+
return (values, labels) if values else None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def schema_fields(schema: Any, resolve: Resolve = unresolved) -> list[FieldType]:
|
|
252
|
+
"""The fields of one entity schema (``properties`` + ``required``), each typed by the JSON
|
|
253
|
+
type words it admits and, when it lists them all, its allowed values: the one reading
|
|
254
|
+
``gateway worlds describe`` reports and every CSV import casts by."""
|
|
255
|
+
node, _ = resolve(schema, frozenset())
|
|
256
|
+
required = set(node.get("required") or [])
|
|
257
|
+
return [
|
|
258
|
+
FieldType(
|
|
259
|
+
name,
|
|
260
|
+
_type_words(schema_types(field_schema, resolve)),
|
|
261
|
+
name in required,
|
|
262
|
+
_allowed_tuple(allowed_values(field_schema, resolve)),
|
|
263
|
+
schema_format(field_schema, resolve),
|
|
264
|
+
)
|
|
265
|
+
for name, field_schema in (node.get("properties") or {}).items()
|
|
266
|
+
]
|
|
184
267
|
|
|
185
268
|
|
|
186
|
-
def
|
|
187
|
-
if
|
|
269
|
+
def _type_words(types: tuple[str, ...]) -> str | None:
|
|
270
|
+
if UNFOLLOWED in types:
|
|
188
271
|
return None
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
values = target.get("enum") if isinstance(target, dict) else None
|
|
195
|
-
return tuple(values) if isinstance(values, list) else None
|
|
272
|
+
return " | ".join(sorted(types, key=lambda word: MEMBERS.index(word) if word in MEMBERS else len(MEMBERS))) or None
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _allowed_tuple(allowed: tuple[list, list[dict]] | None) -> tuple | None:
|
|
276
|
+
return tuple(allowed[0]) if allowed is not None else None
|
|
196
277
|
|
|
197
278
|
|
|
198
|
-
def
|
|
199
|
-
"""
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
279
|
+
def schema_format(schema: Any, resolve: Resolve = unresolved, seen: frozenset = frozenset()) -> str | None:
|
|
280
|
+
"""The ``format`` a schema names, through references and the first ``anyOf`` / ``oneOf`` branch
|
|
281
|
+
that names one."""
|
|
282
|
+
node, seen = resolve(schema, seen)
|
|
283
|
+
if isinstance(node.get("format"), str):
|
|
284
|
+
return node["format"]
|
|
285
|
+
branches = node.get("anyOf") or node.get("oneOf") or []
|
|
286
|
+
return next((found for branch in branches if (found := schema_format(branch, resolve, seen)) is not None), None)
|
|
203
287
|
|
|
204
288
|
|
|
205
289
|
def world_fields(world: dict) -> dict[str, list[FieldType]]:
|
|
206
|
-
"""Every entity's fields as
|
|
207
|
-
|
|
208
|
-
list ``gateway worlds describe`` reports and a CSV import casts by."""
|
|
290
|
+
"""Every entity's fields as an authored world.json declares them, following its own ``$defs``
|
|
291
|
+
only, sorted by entity name: what describe reports when the world's schemas cannot be resolved."""
|
|
209
292
|
gateway = world.get("x-gateway") if isinstance(world, dict) else None
|
|
210
293
|
entities = (gateway.get("entities") or {}) if isinstance(gateway, dict) else {}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
properties = definition.get("properties") or {}
|
|
217
|
-
fields[name] = [
|
|
218
|
-
FieldType(field_name, _field_type(field_schema), field_name in required, _field_enum(field_schema, defs))
|
|
219
|
-
for field_name, field_schema in properties.items()
|
|
220
|
-
]
|
|
221
|
-
return fields
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
def world_casters(world: dict) -> EntityCasters:
|
|
225
|
-
""":func:`casters_of` for a world.json document."""
|
|
226
|
-
return casters_of(world_fields(world))
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
def schema_fields(schema: Any) -> list[FieldType]:
|
|
230
|
-
"""The fields of one resolved entity schema (``properties`` + ``required``)."""
|
|
231
|
-
if not isinstance(schema, dict):
|
|
232
|
-
return []
|
|
233
|
-
required = set(schema.get("required") or [])
|
|
234
|
-
return [
|
|
235
|
-
FieldType(name, _field_type(field_schema), name in required)
|
|
236
|
-
for name, field_schema in (schema.get("properties") or {}).items()
|
|
237
|
-
]
|
|
294
|
+
resolve = LocalDefinitions(world.get("$defs"))
|
|
295
|
+
return {
|
|
296
|
+
name: schema_fields(entity.get("schema") if isinstance(entity, dict) else None, resolve)
|
|
297
|
+
for name, entity in sorted(entities.items())
|
|
298
|
+
}
|
|
238
299
|
|
|
239
300
|
|
|
240
301
|
def contract_casters(entities: dict) -> EntityCasters:
|
|
241
302
|
""":func:`casters_of` for a compiled contract's ``entities`` (each carries its resolved
|
|
242
|
-
``schema``)
|
|
243
|
-
return casters_of(
|
|
303
|
+
``schema``): the local twin of :func:`described_casters`."""
|
|
304
|
+
return casters_of(
|
|
305
|
+
{
|
|
306
|
+
name: schema_fields(entity.get("schema"), LocalDefinitions((entity.get("schema") or {}).get("$defs")))
|
|
307
|
+
for name, entity in entities.items()
|
|
308
|
+
}
|
|
309
|
+
)
|
|
244
310
|
|
|
245
311
|
|
|
246
312
|
def tree_casters(root: str | Path) -> EntityCasters:
|
|
@@ -9,6 +9,12 @@ from pathlib import Path
|
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
11
|
from .._toml import tomllib
|
|
12
|
+
from .shape_declarations import (
|
|
13
|
+
DECLARATION_MEMBERS,
|
|
14
|
+
EntityDeclarations,
|
|
15
|
+
answer_declarations,
|
|
16
|
+
check_generated_sequences,
|
|
17
|
+
)
|
|
12
18
|
from .tree_import import PUBLICATION_FILE_BYTES
|
|
13
19
|
from .validation import (
|
|
14
20
|
FIELD_NAME,
|
|
@@ -21,6 +27,7 @@ from .validation import (
|
|
|
21
27
|
digest,
|
|
22
28
|
encoded,
|
|
23
29
|
fail,
|
|
30
|
+
json_pointer_parts,
|
|
24
31
|
local_definition,
|
|
25
32
|
members,
|
|
26
33
|
nonempty,
|
|
@@ -92,20 +99,6 @@ def quote(value: str) -> str:
|
|
|
92
99
|
return '"' + value.replace('"', '""') + '"'
|
|
93
100
|
|
|
94
101
|
|
|
95
|
-
def json_pointer_parts(value: str, location: str, *, code: str) -> list[str]:
|
|
96
|
-
if not value.startswith("/"):
|
|
97
|
-
return [value]
|
|
98
|
-
parts = []
|
|
99
|
-
for part in value[1:].split("/"):
|
|
100
|
-
if re.search(r"~(?![01])", part):
|
|
101
|
-
fail(code, "Invalid JSON Pointer escape in field path", location)
|
|
102
|
-
decoded = part.replace("~1", "/").replace("~0", "~")
|
|
103
|
-
if '"' in decoded:
|
|
104
|
-
fail(code, "Quoted property names are unsupported in key paths", location)
|
|
105
|
-
parts.append(decoded)
|
|
106
|
-
return parts
|
|
107
|
-
|
|
108
|
-
|
|
109
102
|
def field_schema(schema: dict, field: str, location: str, *, code: str = "key") -> dict:
|
|
110
103
|
parts = json_pointer_parts(field, location, code=code)
|
|
111
104
|
|
|
@@ -469,7 +462,7 @@ def _build(bundle: Path, resolver: Resolver) -> Contract:
|
|
|
469
462
|
seen.add(name.lower())
|
|
470
463
|
members(
|
|
471
464
|
entity,
|
|
472
|
-
{"schema", "primary_key", "unique", "relationships"},
|
|
465
|
+
{"schema", "primary_key", "unique", "relationships", *DECLARATION_MEMBERS},
|
|
473
466
|
{"schema", "primary_key"},
|
|
474
467
|
location,
|
|
475
468
|
)
|
|
@@ -540,6 +533,8 @@ def _build(bundle: Path, resolver: Resolver) -> Contract:
|
|
|
540
533
|
"location": location,
|
|
541
534
|
}
|
|
542
535
|
warnings.extend(cycles.unreached(document.get("$defs", {}), "world.json"))
|
|
536
|
+
payloads = {name: entity["schema"] for name, entity in compiled.items()}
|
|
537
|
+
EntityDeclarations(entities, payloads, "world.json#/x-gateway/entities").check()
|
|
543
538
|
for name, entity in compiled.items():
|
|
544
539
|
for rel in entity["relationships"]:
|
|
545
540
|
location = f"{entity['location']}/relationships"
|
|
@@ -897,12 +892,16 @@ def bind_connection(bundle: Path, contract: Contract) -> None:
|
|
|
897
892
|
spec_path = bundle / "connector.toml"
|
|
898
893
|
if not spec_path.is_file():
|
|
899
894
|
return
|
|
900
|
-
from .connector import
|
|
895
|
+
from .connector import spec_from
|
|
896
|
+
from .validation import parse_toml
|
|
901
897
|
|
|
902
|
-
|
|
898
|
+
raw = parse_toml(spec_path.read_text(encoding="utf-8"))
|
|
899
|
+
spec = spec_from(raw)
|
|
903
900
|
spec.check_entities(contract.entities)
|
|
904
901
|
if spec.webhooks is not None:
|
|
905
902
|
spec.webhooks.check_contract(contract.entities)
|
|
903
|
+
answer_declarations(raw)
|
|
904
|
+
check_generated_sequences(raw, json.loads((bundle / ROOT).read_text(encoding="utf-8")))
|
|
906
905
|
|
|
907
906
|
|
|
908
907
|
def check_bundle(bundle: str | Path) -> Contract:
|
|
@@ -28,6 +28,9 @@ import json
|
|
|
28
28
|
import math
|
|
29
29
|
import re
|
|
30
30
|
import sys
|
|
31
|
+
from collections.abc import Iterable, Iterator
|
|
32
|
+
from contextlib import contextmanager
|
|
33
|
+
from contextvars import ContextVar
|
|
31
34
|
from typing import Any, Callable
|
|
32
35
|
|
|
33
36
|
from .validation import ContractError, fail
|
|
@@ -52,6 +55,7 @@ class Refusal(Exception):
|
|
|
52
55
|
|
|
53
56
|
_DETAIL_LIMIT = 2000
|
|
54
57
|
_FIELD_LIMIT = 256
|
|
58
|
+
ENGINE = "engine"
|
|
55
59
|
_SURROGATE = re.compile("[\ud800-\udfff]")
|
|
56
60
|
|
|
57
61
|
|
|
@@ -89,7 +93,45 @@ def log_contract_refusal(world: str, operation: str, error: ContractError) -> No
|
|
|
89
93
|
|
|
90
94
|
def request_failure_body(error: BaseException) -> dict:
|
|
91
95
|
"""The 500 body for a failure met answering a request, outside any handler."""
|
|
92
|
-
return {"error":
|
|
96
|
+
return {"error": ENGINE, "detail": f"{type(error).__name__}: {_detail(error)}"[:_DETAIL_LIMIT]}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class OutOfMemory:
|
|
100
|
+
"""A request whose world code raised ``MemoryError``, or ``SystemError`` (what C code raises when an
|
|
101
|
+
allocation fails without setting one), noted in that request's own context where the raise becomes
|
|
102
|
+
its answer (:func:`failure_answer`, the host's request handler) and never read from the answer, so
|
|
103
|
+
no data a world serves can claim one. A worker's front end says so to its supervisor in ``HEADER``
|
|
104
|
+
on that answer, and sends no other header of that name; the supervisor recycles the worker only
|
|
105
|
+
when the kernel counts it near its cap."""
|
|
106
|
+
|
|
107
|
+
HEADER = "X-Gateway-Out-Of-Memory"
|
|
108
|
+
_noted: ContextVar[bool] = ContextVar("gateway_out_of_memory", default=False)
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
@contextmanager
|
|
112
|
+
def request(cls) -> Iterator[None]:
|
|
113
|
+
token = cls._noted.set(False)
|
|
114
|
+
try:
|
|
115
|
+
yield
|
|
116
|
+
finally:
|
|
117
|
+
cls._noted.reset(token)
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def note(cls, error: BaseException) -> None:
|
|
121
|
+
if isinstance(error, (MemoryError, SystemError)):
|
|
122
|
+
cls._noted.set(True)
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
def noted(cls) -> bool:
|
|
126
|
+
return cls._noted.get()
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def named(cls, header: str) -> bool:
|
|
130
|
+
return header.lower() == cls.HEADER.lower()
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def in_headers(cls, headers: Iterable[tuple[str, str]]) -> bool:
|
|
134
|
+
return any(cls.named(name) for name, _ in headers)
|
|
93
135
|
|
|
94
136
|
|
|
95
137
|
def engine_failure_body(handler: str, error: BaseException) -> dict:
|
|
@@ -99,6 +141,7 @@ def engine_failure_body(handler: str, error: BaseException) -> dict:
|
|
|
99
141
|
|
|
100
142
|
def failure_answer(handler: str, error: Exception) -> tuple[int, dict]:
|
|
101
143
|
"""A handler's raise as its ``(status, body)``: a refusal, its author's bug or the engine's failure."""
|
|
144
|
+
OutOfMemory.note(error)
|
|
102
145
|
if isinstance(error, Refusal):
|
|
103
146
|
return refusal_answer(handler, error)
|
|
104
147
|
if isinstance(error, HANDLER_BUG_ERRORS):
|
|
@@ -127,13 +170,24 @@ def unencodable(body: Any) -> Exception | None:
|
|
|
127
170
|
return None
|
|
128
171
|
|
|
129
172
|
|
|
173
|
+
_HEADER_NAME = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
|
|
174
|
+
_LINE_BREAK = re.compile(r"[\r\n\x00]")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def header_line(name: str, value: str) -> bool:
|
|
178
|
+
"""Whether ``name: value`` is one header line: a token for a name, no CR, LF or NUL in the value, so no
|
|
179
|
+
answer can add a line of its own (a forged ``OutOfMemory.HEADER`` included)."""
|
|
180
|
+
return bool(_HEADER_NAME.fullmatch(name)) and not _LINE_BREAK.search(value)
|
|
181
|
+
|
|
182
|
+
|
|
130
183
|
def _unsendable_header(headers: dict[str, str] | None) -> Exception | None:
|
|
131
184
|
try:
|
|
132
185
|
for name, value in (headers or {}).items():
|
|
133
186
|
f"{name}: {value}".encode("latin-1")
|
|
134
187
|
except UnicodeEncodeError as error:
|
|
135
|
-
return error
|
|
136
|
-
|
|
188
|
+
return ValueError(f"a header is not latin-1: {error}")
|
|
189
|
+
split = next((name for name, value in (headers or {}).items() if not header_line(name, value)), None)
|
|
190
|
+
return None if split is None else ValueError(f"header {split!r} is not one line: a token name, no CR, LF or NUL")
|
|
137
191
|
|
|
138
192
|
|
|
139
193
|
def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None:
|
|
@@ -143,7 +197,7 @@ def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None
|
|
|
143
197
|
return 400, handler_bug_body(handler, ValueError(f"an answer names a status from 200 to 599, not {status!r}"))
|
|
144
198
|
header = _unsendable_header(headers)
|
|
145
199
|
if header is not None:
|
|
146
|
-
return 400, handler_bug_body(handler, header
|
|
200
|
+
return 400, handler_bug_body(handler, header)
|
|
147
201
|
error = unencodable(body)
|
|
148
202
|
return None if error is None else (400, handler_bug_body(handler, error, "the answer is not JSON: "))
|
|
149
203
|
|
|
@@ -60,7 +60,7 @@ from .call_capture import JSON_TYPE, send_response
|
|
|
60
60
|
from .call_redaction import CallRedaction
|
|
61
61
|
from .credentials import BearerAuth
|
|
62
62
|
from .door import PLATFORM_HEADER, Entry
|
|
63
|
-
from .handler_errors import log_request_failure, request_failure_body
|
|
63
|
+
from .handler_errors import OutOfMemory, header_line, log_request_failure, request_failure_body
|
|
64
64
|
from .host_browser import RelayNavigation, log_relay
|
|
65
65
|
from .host_control import ControlPlane
|
|
66
66
|
from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
|
|
@@ -100,6 +100,7 @@ from .workers import (
|
|
|
100
100
|
DEFAULT_WORKER_BUDGET_MB,
|
|
101
101
|
DEFAULT_WORKER_FILE_MB,
|
|
102
102
|
DEFAULT_WORKER_FOOTPRINT_MB,
|
|
103
|
+
DEFAULT_WORKER_MEMORY_CEILING_MB,
|
|
103
104
|
DEFAULT_WORKER_MEMORY_MB,
|
|
104
105
|
WorkerPool,
|
|
105
106
|
seal_process,
|
|
@@ -180,6 +181,20 @@ def make_host_handler(
|
|
|
180
181
|
return {k.lower(): v for k, v in self.headers.items()}
|
|
181
182
|
|
|
182
183
|
def _dispatch(self, method: str) -> None:
|
|
184
|
+
with OutOfMemory.request():
|
|
185
|
+
self._answer(method)
|
|
186
|
+
|
|
187
|
+
def send_header(self, keyword: str, value: str) -> None:
|
|
188
|
+
if OutOfMemory.named(keyword) or not header_line(keyword, str(value)):
|
|
189
|
+
return
|
|
190
|
+
super().send_header(keyword, value)
|
|
191
|
+
|
|
192
|
+
def end_headers(self) -> None:
|
|
193
|
+
if host.run_handlers and OutOfMemory.noted():
|
|
194
|
+
super().send_header(OutOfMemory.HEADER, "1")
|
|
195
|
+
super().end_headers()
|
|
196
|
+
|
|
197
|
+
def _answer(self, method: str) -> None:
|
|
183
198
|
self._body: bytes | None = None
|
|
184
199
|
url = urllib.parse.urlsplit(self.path)
|
|
185
200
|
if url.path == "/healthz" and control.internal(self.path, self.headers):
|
|
@@ -203,6 +218,7 @@ def make_host_handler(
|
|
|
203
218
|
self.close_connection = True
|
|
204
219
|
return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
|
|
205
220
|
except Exception as error:
|
|
221
|
+
OutOfMemory.note(error)
|
|
206
222
|
log_request_failure(error, method, url.path)
|
|
207
223
|
return self._reply(500, request_failure_body(error))
|
|
208
224
|
|
|
@@ -512,7 +528,8 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
512
528
|
``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
|
|
513
529
|
all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
|
|
514
530
|
its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
|
|
515
|
-
``
|
|
531
|
+
``WORLDS_WORKER_MEMORY_CEILING_MB`` the most address space a worker measured to need more is given (unset,
|
|
532
|
+
the cap), ``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
|
|
516
533
|
``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
|
|
517
534
|
host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
|
|
518
535
|
``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
|
|
@@ -540,6 +557,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
540
557
|
lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
|
|
541
558
|
workers = None
|
|
542
559
|
footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
|
|
560
|
+
ceiling = os.environ.get("WORLDS_WORKER_MEMORY_CEILING_MB")
|
|
543
561
|
if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
|
|
544
562
|
workers = WorkerPool(
|
|
545
563
|
Path(cache_root),
|
|
@@ -548,6 +566,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
548
566
|
budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
|
|
549
567
|
footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
|
|
550
568
|
file_mb=int(os.environ.get("WORLDS_WORKER_FILE_MB") or DEFAULT_WORKER_FILE_MB),
|
|
569
|
+
ceiling_mb=int(ceiling) if ceiling else DEFAULT_WORKER_MEMORY_CEILING_MB,
|
|
551
570
|
)
|
|
552
571
|
owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
|
|
553
572
|
world_host = WorldHost(
|
|
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any
|
|
|
16
16
|
|
|
17
17
|
from .handler_errors import carried
|
|
18
18
|
from .host_browser import RelayNavigation
|
|
19
|
-
from .host_requests import BrowserRelay
|
|
19
|
+
from .host_requests import BrowserRelay, parse_memory_peak, parse_world_state
|
|
20
20
|
from .host_surfaces import public_host
|
|
21
21
|
from .host_workers import WORKER_EXITED, refusal
|
|
22
22
|
from .validation import ContractError, fail
|
|
@@ -148,6 +148,8 @@ class ControlPlane:
|
|
|
148
148
|
auth=body.get("auth"),
|
|
149
149
|
ledger_source=body.get("ledgerSource"),
|
|
150
150
|
parameters=body.get("parameters"),
|
|
151
|
+
world_state=parse_world_state(body.get("worldState")),
|
|
152
|
+
memory_peak_mb=parse_memory_peak(body.get("memoryPeakMb")),
|
|
151
153
|
)
|
|
152
154
|
|
|
153
155
|
def _swap_browser(self, session_id: str, browser: Any) -> dict:
|
|
@@ -4,6 +4,7 @@ platform what the host is and holds. A step that fails is logged and the next on
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import http.client
|
|
7
8
|
import json
|
|
8
9
|
import threading
|
|
9
10
|
import urllib.error
|
|
@@ -18,6 +19,8 @@ if TYPE_CHECKING:
|
|
|
18
19
|
|
|
19
20
|
LEASE_SWEEP_S = 30.0
|
|
20
21
|
HEARTBEAT_S = 10.0
|
|
22
|
+
#: The most a heartbeat reads of the platform's answer.
|
|
23
|
+
ANSWER_LIMIT_BYTES = 64 << 10
|
|
21
24
|
|
|
22
25
|
|
|
23
26
|
def swept(step: Callable[[], list[str]], done: str) -> list[str]:
|
|
@@ -49,7 +52,8 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
|
|
|
49
52
|
"""Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
|
|
50
53
|
The platform's registry is what places new sessions: a host it has not
|
|
51
54
|
heard from lately gets none. Returns whether the platform took it."""
|
|
52
|
-
|
|
55
|
+
stats = world_host.stats()
|
|
56
|
+
body = json.dumps(stats).encode("utf-8")
|
|
53
57
|
req = urllib.request.Request(
|
|
54
58
|
f"{platform_url.rstrip('/')}/api/public/world-hosts",
|
|
55
59
|
data=body,
|
|
@@ -58,10 +62,23 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
|
|
|
58
62
|
)
|
|
59
63
|
try:
|
|
60
64
|
with urllib.request.urlopen(req, timeout=timeout_s) as r:
|
|
61
|
-
|
|
62
|
-
|
|
65
|
+
taken = 200 <= r.status < 300
|
|
66
|
+
peaks_taken = taken and _peaks_taken(r.read(ANSWER_LIMIT_BYTES))
|
|
67
|
+
except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
|
|
63
68
|
log_line(f"heartbeat failed: {error}")
|
|
64
69
|
return False
|
|
70
|
+
if peaks_taken:
|
|
71
|
+
world_host.reported(stats)
|
|
72
|
+
return taken
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _peaks_taken(answer: bytes) -> bool:
|
|
76
|
+
"""Whether the platform says it kept the beat's worker peaks: one that predates them takes the beat
|
|
77
|
+
and ignores them, so they stay pending until a platform that keeps them answers."""
|
|
78
|
+
try:
|
|
79
|
+
return json.loads(answer).get("peaksTaken") is True
|
|
80
|
+
except (ValueError, AttributeError):
|
|
81
|
+
return False
|
|
65
82
|
|
|
66
83
|
|
|
67
84
|
class Heartbeat:
|
|
@@ -90,8 +107,14 @@ class Heartbeat:
|
|
|
90
107
|
self._due.set()
|
|
91
108
|
|
|
92
109
|
def _beat(self) -> None:
|
|
93
|
-
|
|
110
|
+
self._once()
|
|
94
111
|
while not self._stopped.is_set():
|
|
95
112
|
self._due.wait(self.interval_s)
|
|
96
113
|
self._due.clear()
|
|
114
|
+
self._once()
|
|
115
|
+
|
|
116
|
+
def _once(self) -> None:
|
|
117
|
+
try:
|
|
97
118
|
heartbeat_once(self.world_host, self.platform_url, self.secret)
|
|
119
|
+
except Exception as error: # noqa: BLE001 - one failed beat must not stop the beats after it
|
|
120
|
+
log_line(f"heartbeat failed: {type(error).__name__}")
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""The prometheus exposition of a World Host's :meth:`WorldHost.stats`. ``worlds_host_sessions`` is the
|
|
2
|
+
number the fleet scales on (sessions per host, not cpu)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from .host_handoff import OUTCOMES
|
|
7
|
+
from .host_workers import WORKERS_POISONED
|
|
8
|
+
from .worker_watch import WORKERS_NEAR_CAP
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _largest_share(workers: list[dict]) -> list[str]:
|
|
12
|
+
"""The most of its own cap any worker holds (``VmSize`` over ``RLIMIT_AS``), no sample when none was
|
|
13
|
+
measured. A total: world code on the pod can read ``/metrics``, so no line here names a version."""
|
|
14
|
+
shares = [w["sizeMb"] / w["capMb"] for w in workers if w.get("sizeMb") is not None and w.get("capMb")]
|
|
15
|
+
sample = [f"worlds_host_worker_cap_share_max {max(shares):.3f}"] if shares else []
|
|
16
|
+
return ["# TYPE worlds_host_worker_cap_share_max gauge", *sample]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def exposition(stats: dict) -> str:
|
|
20
|
+
counters: dict[str, int] = stats["counters"]
|
|
21
|
+
lines = [
|
|
22
|
+
"# TYPE worlds_host_sessions gauge",
|
|
23
|
+
f"worlds_host_sessions {stats['sessions']}",
|
|
24
|
+
"# TYPE worlds_host_capacity gauge",
|
|
25
|
+
f"worlds_host_capacity {stats['capacity']}",
|
|
26
|
+
"# TYPE worlds_host_draining gauge",
|
|
27
|
+
f"worlds_host_draining {int(stats['draining'])}",
|
|
28
|
+
"# TYPE worlds_host_workers gauge",
|
|
29
|
+
f"worlds_host_workers {stats['workers']}",
|
|
30
|
+
"# TYPE worlds_host_worker_memory_mb gauge",
|
|
31
|
+
f"worlds_host_worker_memory_mb {stats['workerMemoryMb']}",
|
|
32
|
+
"# TYPE worlds_host_versions_cached gauge",
|
|
33
|
+
f"worlds_host_versions_cached {len(stats['versions'])}",
|
|
34
|
+
"# TYPE worlds_host_uptime_seconds gauge",
|
|
35
|
+
f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
|
|
36
|
+
"# TYPE worlds_host_sessions_opened_total counter",
|
|
37
|
+
f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
|
|
38
|
+
"# TYPE worlds_host_open_failures_total counter",
|
|
39
|
+
f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
|
|
40
|
+
"# TYPE worlds_host_session_seconds_total counter",
|
|
41
|
+
f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
|
|
42
|
+
"# TYPE worlds_host_exports_total counter",
|
|
43
|
+
f"worlds_host_exports_total {counters.get('exports', 0)}",
|
|
44
|
+
"# TYPE worlds_host_exports_uploaded_total counter",
|
|
45
|
+
f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
|
|
46
|
+
"# TYPE worlds_host_sessions_closed_total counter",
|
|
47
|
+
]
|
|
48
|
+
for reason in ("closed", "expired", "lost", "replaced", "moved"):
|
|
49
|
+
lines.append(f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}')
|
|
50
|
+
lines.append("# TYPE worlds_host_handoffs_total counter")
|
|
51
|
+
for outcome in OUTCOMES:
|
|
52
|
+
lines.append(f'worlds_host_handoffs_total{{outcome="{outcome}"}} {counters.get(f"handoffs.{outcome}", 0)}')
|
|
53
|
+
workers = stats.get("workerMemory") or []
|
|
54
|
+
lines += _largest_share(workers)
|
|
55
|
+
lines += [
|
|
56
|
+
"# TYPE worlds_host_workers_near_cap gauge",
|
|
57
|
+
f"worlds_host_workers_near_cap {sum(1 for w in workers if w.get('nearCap'))}",
|
|
58
|
+
]
|
|
59
|
+
recoveries = ("workers_replaced", "workers_recycled", WORKERS_POISONED, WORKERS_NEAR_CAP, "sessions_recovered")
|
|
60
|
+
for name in (*recoveries, "sessions_unrecovered"):
|
|
61
|
+
lines += [f"# TYPE worlds_host_{name}_total counter", f"worlds_host_{name}_total {counters.get(name, 0)}"]
|
|
62
|
+
lines.append("# TYPE worlds_host_requests_total counter")
|
|
63
|
+
for surface in ("api", "ui", "browser", "proxied"):
|
|
64
|
+
lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
|
|
65
|
+
lines += [
|
|
66
|
+
"# TYPE worlds_host_platform_bearer_requests_total counter",
|
|
67
|
+
f"worlds_host_platform_bearer_requests_total {counters.get('requests.platform_bearer', 0)}",
|
|
68
|
+
]
|
|
69
|
+
return "\n".join(lines) + "\n"
|