@withgateway/sdk 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/agents/index.d.mts +12 -4
  2. package/dist/agents/index.d.ts +12 -4
  3. package/dist/agents/index.js +30 -0
  4. package/dist/agents/index.js.map +1 -1
  5. package/dist/agents/index.mjs +30 -0
  6. package/dist/agents/index.mjs.map +1 -1
  7. package/dist/cli.js +2291 -945
  8. package/dist/cli.js.map +1 -1
  9. package/dist/cli.mjs +2290 -944
  10. package/dist/cli.mjs.map +1 -1
  11. package/dist/files/index.js.map +1 -1
  12. package/dist/files/index.mjs.map +1 -1
  13. package/dist/index.js +325 -184
  14. package/dist/index.js.map +1 -1
  15. package/dist/index.mjs +324 -183
  16. package/dist/index.mjs.map +1 -1
  17. package/dist/traces/index.js.map +1 -1
  18. package/dist/traces/index.mjs.map +1 -1
  19. package/dist/tracing/index.js +1 -1
  20. package/dist/tracing/index.js.map +1 -1
  21. package/dist/tracing/index.mjs +1 -1
  22. package/dist/tracing/index.mjs.map +1 -1
  23. package/dist/worlds/index.d.mts +3 -1
  24. package/dist/worlds/index.d.ts +3 -1
  25. package/dist/worlds/index.js +178 -67
  26. package/dist/worlds/index.js.map +1 -1
  27. package/dist/worlds/index.mjs +177 -66
  28. package/dist/worlds/index.mjs.map +1 -1
  29. package/package.json +1 -1
  30. package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
  31. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  32. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  33. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  34. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  35. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  36. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  37. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  38. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  39. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  40. package/runtime/gatewaysdk/environment/schema/host.py +21 -2
  41. package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
  42. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  43. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  44. package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
  45. package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
  46. package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
  47. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  48. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  49. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  51. package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
  52. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  53. package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
  54. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  55. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  56. package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
  57. package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
  58. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  59. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  60. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  61. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
  62. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  63. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  64. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
  65. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
  66. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  68. package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
  69. package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
  70. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  71. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  72. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  73. package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
  74. package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
  75. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  76. package/skills/connector-schema-authoring/SKILL.md +1 -0
  77. package/skills/connector-schema-authoring/connector-worlds.md +1 -0
  78. package/skills/connector-schema-authoring/world-schema.md +7 -0
  79. package/skills/factory-orchestrator/SKILL.md +147 -0
  80. package/skills/world-data-ingestion/ROWS.md +5 -1
  81. package/skills/world-data-ingestion/SKILL.md +4 -4
  82. package/skills/worlds-getting-started/SKILL.md +2 -2
  83. package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  84. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
@@ -23,6 +23,8 @@ from dataclasses import dataclass
23
23
  from pathlib import Path
24
24
  from typing import Any
25
25
 
26
+ from .validation import local_key
27
+
26
28
  INTEGER_TEXT = re.compile(r"^[+-]?\d+$")
27
29
  NUMBER_TEXT = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
28
30
  BOOLEANS = {"true": True, "false": False, "1": True, "0": False}
@@ -33,19 +35,23 @@ _MISSING = object()
33
35
 
34
36
  @dataclass(frozen=True)
35
37
  class FieldType:
36
- """One field as ``gateway worlds describe`` reports it: ``type`` is the JSON Schema
37
- type word(s) joined by ``" | "``, a ``$ref`` name, or None when the schema says nothing;
38
- ``enum`` the values the schema allows, when it lists them."""
38
+ """One field as ``gateway worlds describe`` reports it: ``type`` is the JSON Schema type
39
+ words the field admits, through references and unions, joined by ``" | "``, or None when the
40
+ schema says nothing; ``enum`` the values it allows, when it lists every one; ``format`` the
41
+ schema's ``format`` (``date-time``, ``email``, ...), when it names one."""
39
42
 
40
43
  name: str
41
44
  type: str | None
42
45
  required: bool
43
46
  enum: tuple | None = None
47
+ format: str | None = None
44
48
 
45
49
  def describe(self) -> dict:
46
50
  described: dict = {"name": self.name, "type": self.type, "required": self.required}
47
51
  if self.enum is not None:
48
52
  described["enum"] = list(self.enum)
53
+ if self.format is not None:
54
+ described["format"] = self.format
49
55
  return described
50
56
 
51
57
 
@@ -160,87 +166,147 @@ def casters_of(entities: dict[str, list[FieldType]]) -> EntityCasters:
160
166
  return by_name.get
161
167
 
162
168
 
163
- def _definition(world: dict, entity: Any) -> dict:
164
- schema = entity.get("schema") if isinstance(entity, dict) else None
165
- ref = schema.get("$ref") if isinstance(schema, dict) else None
166
- if isinstance(ref, str) and ref.startswith("#/$defs/"):
167
- definition = (world.get("$defs") or {}).get(ref[len("#/$defs/") :])
168
- return definition if isinstance(definition, dict) else {}
169
- if isinstance(schema, dict) and isinstance(schema.get("properties"), dict):
170
- return schema
171
- return {}
169
+ Resolve = Callable[[Any, frozenset], tuple[dict, frozenset]]
170
+ UNFOLLOWED = "$ref"
171
+
172
+
173
+ def unresolved(schema: Any, seen: frozenset) -> tuple[dict, frozenset]:
174
+ return (schema if isinstance(schema, dict) else {}), seen
175
+
176
+
177
+ class LocalDefinitions:
178
+ """Follows ``#/$defs/<key>`` references into ``defs``; ``seen`` carries the keys on the path,
179
+ so a cycle through compositions ends."""
180
+
181
+ def __init__(self, defs: Any):
182
+ self.defs = defs if isinstance(defs, dict) else {}
183
+
184
+ def __call__(self, schema: Any, seen: frozenset = frozenset()) -> tuple[dict, frozenset]:
185
+ while isinstance(schema, dict):
186
+ key = local_key(schema.get("$ref"))
187
+ if key is None or key in seen or not isinstance(self.defs.get(key), dict):
188
+ break
189
+ seen = seen | {key}
190
+ schema = self.defs[key]
191
+ return unresolved(schema, seen)
192
+
193
+
194
+ def schema_types(schema: Any, resolve: Resolve = unresolved, seen: frozenset = frozenset()) -> tuple[str, ...]:
195
+ """The JSON type words a schema admits, in the order it declares them, through references
196
+ and ``anyOf`` / ``oneOf`` / ``allOf``; :data:`UNFOLLOWED` for a reference ``resolve`` cannot follow
197
+ that carries no ``type`` of its own (a shared compile keeps the target's ``type`` beside its ``$ref``)."""
198
+ node, seen = resolve(schema, seen)
199
+ if isinstance(node.get("$ref"), str) and local_key(node["$ref"]) not in seen and "type" not in node:
200
+ return (UNFOLLOWED,)
201
+ kind = node.get("type")
202
+ found = [kind] if isinstance(kind, str) else [part for part in kind or [] if isinstance(part, str)]
203
+ for key in ("anyOf", "oneOf", "allOf"):
204
+ for branch in node.get(key) or []:
205
+ found += schema_types(branch, resolve, seen)
206
+ return tuple(dict.fromkeys(found))
207
+
208
+
209
+ def allowed_values(
210
+ schema: Any, resolve: Resolve = unresolved, seen: frozenset = frozenset()
211
+ ) -> tuple[list, list[dict]] | None:
212
+ """The values a schema allows when it lists every one (``enum``, ``const``, a ``oneOf`` /
213
+ ``anyOf`` of those with a ``null`` branch allowing None, or the values every listing ``allOf``
214
+ branch shares), with each labelled ``const``'s ``title``; None when it also admits values it
215
+ does not list."""
216
+ node, seen = resolve(schema, seen)
217
+ if isinstance(node.get("enum"), list):
218
+ return list(node["enum"]), []
219
+ if "const" in node:
220
+ label = node.get("title")
221
+ return [node["const"]], ([{"value": node["const"], "label": label}] if isinstance(label, str) else [])
222
+ if node.get("allOf") and not (node.get("oneOf") or node.get("anyOf")):
223
+ return _shared_values(node["allOf"], resolve, seen)
224
+ branches = node.get("oneOf") or node.get("anyOf")
225
+ if not isinstance(branches, list) or not branches:
226
+ return None
227
+ values: list = []
228
+ labels: list[dict] = []
229
+ for branch in branches:
230
+ if schema_types(branch, resolve, seen) == ("null",):
231
+ values.append(None)
232
+ continue
233
+ allowed = allowed_values(branch, resolve, seen)
234
+ if allowed is None:
235
+ return None
236
+ values += allowed[0]
237
+ labels += allowed[1]
238
+ return (values, labels) if [value for value in values if value is not None] else None
172
239
 
173
240
 
174
- def _field_type(field_schema: Any) -> str | None:
175
- if not isinstance(field_schema, dict):
241
+ def _shared_values(branches: list, resolve: Resolve, seen: frozenset) -> tuple[list, list[dict]] | None:
242
+ listed = [allowed for branch in branches if (allowed := allowed_values(branch, resolve, seen)) is not None]
243
+ if not listed:
176
244
  return None
177
- kind = field_schema.get("type")
178
- if isinstance(kind, list):
179
- # A nullable field is `["string", "null"]` in JSON Schema; one readable type, not the list.
180
- return " | ".join(str(part) for part in kind)
181
- if kind is None and isinstance(field_schema.get("$ref"), str):
182
- return field_schema["$ref"].rsplit("/", 1)[-1]
183
- return kind if isinstance(kind, str) else None
245
+ values = [value for value in listed[0][0] if all(value in other[0] for other in listed[1:])]
246
+ labels = [label for _, branch_labels in listed for label in branch_labels if label["value"] in values]
247
+ labels = [label for index, label in enumerate(labels) if label not in labels[:index]]
248
+ return (values, labels) if values else None
249
+
250
+
251
+ def schema_fields(schema: Any, resolve: Resolve = unresolved) -> list[FieldType]:
252
+ """The fields of one entity schema (``properties`` + ``required``), each typed by the JSON
253
+ type words it admits and, when it lists them all, its allowed values: the one reading
254
+ ``gateway worlds describe`` reports and every CSV import casts by."""
255
+ node, _ = resolve(schema, frozenset())
256
+ required = set(node.get("required") or [])
257
+ return [
258
+ FieldType(
259
+ name,
260
+ _type_words(schema_types(field_schema, resolve)),
261
+ name in required,
262
+ _allowed_tuple(allowed_values(field_schema, resolve)),
263
+ schema_format(field_schema, resolve),
264
+ )
265
+ for name, field_schema in (node.get("properties") or {}).items()
266
+ ]
184
267
 
185
268
 
186
- def _field_enum(field_schema: Any, defs: dict) -> tuple | None:
187
- if not isinstance(field_schema, dict):
269
+ def _type_words(types: tuple[str, ...]) -> str | None:
270
+ if UNFOLLOWED in types:
188
271
  return None
189
- branches = field_schema.get("anyOf") or field_schema.get("oneOf")
190
- if isinstance(branches, list):
191
- return _branches_enum(branches, defs)
192
- ref = field_schema.get("$ref")
193
- target = defs.get(ref[len("#/$defs/") :]) if isinstance(ref, str) and ref.startswith("#/$defs/") else field_schema
194
- values = target.get("enum") if isinstance(target, dict) else None
195
- return tuple(values) if isinstance(values, list) else None
272
+ return " | ".join(sorted(types, key=lambda word: MEMBERS.index(word) if word in MEMBERS else len(MEMBERS))) or None
273
+
274
+
275
+ def _allowed_tuple(allowed: tuple[list, list[dict]] | None) -> tuple | None:
276
+ return tuple(allowed[0]) if allowed is not None else None
196
277
 
197
278
 
198
- def _branches_enum(branches: list, defs: dict) -> tuple | None:
199
- """A nullable enum (``anyOf: [{$ref: Enum}, {type: null}]``) allows the enum's values and null."""
200
- values = [value for branch in branches for value in (_field_enum(branch, defs) or ())]
201
- nullable = any(isinstance(branch, dict) and branch.get("type") == "null" for branch in branches)
202
- return tuple(values) + ((None,) if nullable else ()) if values else None
279
+ def schema_format(schema: Any, resolve: Resolve = unresolved, seen: frozenset = frozenset()) -> str | None:
280
+ """The ``format`` a schema names, through references and the first ``anyOf`` / ``oneOf`` branch
281
+ that names one."""
282
+ node, seen = resolve(schema, seen)
283
+ if isinstance(node.get("format"), str):
284
+ return node["format"]
285
+ branches = node.get("anyOf") or node.get("oneOf") or []
286
+ return next((found for branch in branches if (found := schema_format(branch, resolve, seen)) is not None), None)
203
287
 
204
288
 
205
289
  def world_fields(world: dict) -> dict[str, list[FieldType]]:
206
- """Every entity's fields as a world.json document declares them (the ``$defs`` entry its
207
- ``schema.$ref`` names, or an inline ``properties``), sorted by entity name — the field
208
- list ``gateway worlds describe`` reports and a CSV import casts by."""
290
+ """Every entity's fields as an authored world.json declares them, following its own ``$defs``
291
+ only, sorted by entity name: what describe reports when the world's schemas cannot be resolved."""
209
292
  gateway = world.get("x-gateway") if isinstance(world, dict) else None
210
293
  entities = (gateway.get("entities") or {}) if isinstance(gateway, dict) else {}
211
- defs = world.get("$defs") or {}
212
- fields: dict[str, list[FieldType]] = {}
213
- for name, entity in sorted(entities.items()):
214
- definition = _definition(world, entity)
215
- required = set(definition.get("required") or [])
216
- properties = definition.get("properties") or {}
217
- fields[name] = [
218
- FieldType(field_name, _field_type(field_schema), field_name in required, _field_enum(field_schema, defs))
219
- for field_name, field_schema in properties.items()
220
- ]
221
- return fields
222
-
223
-
224
- def world_casters(world: dict) -> EntityCasters:
225
- """:func:`casters_of` for a world.json document."""
226
- return casters_of(world_fields(world))
227
-
228
-
229
- def schema_fields(schema: Any) -> list[FieldType]:
230
- """The fields of one resolved entity schema (``properties`` + ``required``)."""
231
- if not isinstance(schema, dict):
232
- return []
233
- required = set(schema.get("required") or [])
234
- return [
235
- FieldType(name, _field_type(field_schema), name in required)
236
- for name, field_schema in (schema.get("properties") or {}).items()
237
- ]
294
+ resolve = LocalDefinitions(world.get("$defs"))
295
+ return {
296
+ name: schema_fields(entity.get("schema") if isinstance(entity, dict) else None, resolve)
297
+ for name, entity in sorted(entities.items())
298
+ }
238
299
 
239
300
 
240
301
  def contract_casters(entities: dict) -> EntityCasters:
241
302
  """:func:`casters_of` for a compiled contract's ``entities`` (each carries its resolved
242
- ``schema``) — the local twin of :func:`world_casters`."""
243
- return casters_of({name: schema_fields(entity.get("schema")) for name, entity in entities.items()})
303
+ ``schema``): the local twin of :func:`described_casters`."""
304
+ return casters_of(
305
+ {
306
+ name: schema_fields(entity.get("schema"), LocalDefinitions((entity.get("schema") or {}).get("$defs")))
307
+ for name, entity in entities.items()
308
+ }
309
+ )
244
310
 
245
311
 
246
312
  def tree_casters(root: str | Path) -> EntityCasters:
@@ -9,6 +9,12 @@ from pathlib import Path
9
9
  from typing import Any
10
10
 
11
11
  from .._toml import tomllib
12
+ from .shape_declarations import (
13
+ DECLARATION_MEMBERS,
14
+ EntityDeclarations,
15
+ answer_declarations,
16
+ check_generated_sequences,
17
+ )
12
18
  from .tree_import import PUBLICATION_FILE_BYTES
13
19
  from .validation import (
14
20
  FIELD_NAME,
@@ -21,6 +27,7 @@ from .validation import (
21
27
  digest,
22
28
  encoded,
23
29
  fail,
30
+ json_pointer_parts,
24
31
  local_definition,
25
32
  members,
26
33
  nonempty,
@@ -92,20 +99,6 @@ def quote(value: str) -> str:
92
99
  return '"' + value.replace('"', '""') + '"'
93
100
 
94
101
 
95
- def json_pointer_parts(value: str, location: str, *, code: str) -> list[str]:
96
- if not value.startswith("/"):
97
- return [value]
98
- parts = []
99
- for part in value[1:].split("/"):
100
- if re.search(r"~(?![01])", part):
101
- fail(code, "Invalid JSON Pointer escape in field path", location)
102
- decoded = part.replace("~1", "/").replace("~0", "~")
103
- if '"' in decoded:
104
- fail(code, "Quoted property names are unsupported in key paths", location)
105
- parts.append(decoded)
106
- return parts
107
-
108
-
109
102
  def field_schema(schema: dict, field: str, location: str, *, code: str = "key") -> dict:
110
103
  parts = json_pointer_parts(field, location, code=code)
111
104
 
@@ -469,7 +462,7 @@ def _build(bundle: Path, resolver: Resolver) -> Contract:
469
462
  seen.add(name.lower())
470
463
  members(
471
464
  entity,
472
- {"schema", "primary_key", "unique", "relationships"},
465
+ {"schema", "primary_key", "unique", "relationships", *DECLARATION_MEMBERS},
473
466
  {"schema", "primary_key"},
474
467
  location,
475
468
  )
@@ -540,6 +533,8 @@ def _build(bundle: Path, resolver: Resolver) -> Contract:
540
533
  "location": location,
541
534
  }
542
535
  warnings.extend(cycles.unreached(document.get("$defs", {}), "world.json"))
536
+ payloads = {name: entity["schema"] for name, entity in compiled.items()}
537
+ EntityDeclarations(entities, payloads, "world.json#/x-gateway/entities").check()
543
538
  for name, entity in compiled.items():
544
539
  for rel in entity["relationships"]:
545
540
  location = f"{entity['location']}/relationships"
@@ -897,12 +892,16 @@ def bind_connection(bundle: Path, contract: Contract) -> None:
897
892
  spec_path = bundle / "connector.toml"
898
893
  if not spec_path.is_file():
899
894
  return
900
- from .connector import parse_spec
895
+ from .connector import spec_from
896
+ from .validation import parse_toml
901
897
 
902
- spec = parse_spec(spec_path.read_text(encoding="utf-8"))
898
+ raw = parse_toml(spec_path.read_text(encoding="utf-8"))
899
+ spec = spec_from(raw)
903
900
  spec.check_entities(contract.entities)
904
901
  if spec.webhooks is not None:
905
902
  spec.webhooks.check_contract(contract.entities)
903
+ answer_declarations(raw)
904
+ check_generated_sequences(raw, json.loads((bundle / ROOT).read_text(encoding="utf-8")))
906
905
 
907
906
 
908
907
  def check_bundle(bundle: str | Path) -> Contract:
@@ -28,6 +28,9 @@ import json
28
28
  import math
29
29
  import re
30
30
  import sys
31
+ from collections.abc import Iterable, Iterator
32
+ from contextlib import contextmanager
33
+ from contextvars import ContextVar
31
34
  from typing import Any, Callable
32
35
 
33
36
  from .validation import ContractError, fail
@@ -52,6 +55,7 @@ class Refusal(Exception):
52
55
 
53
56
  _DETAIL_LIMIT = 2000
54
57
  _FIELD_LIMIT = 256
58
+ ENGINE = "engine"
55
59
  _SURROGATE = re.compile("[\ud800-\udfff]")
56
60
 
57
61
 
@@ -89,7 +93,45 @@ def log_contract_refusal(world: str, operation: str, error: ContractError) -> No
89
93
 
90
94
  def request_failure_body(error: BaseException) -> dict:
91
95
  """The 500 body for a failure met answering a request, outside any handler."""
92
- return {"error": "engine", "detail": f"{type(error).__name__}: {_detail(error)}"[:_DETAIL_LIMIT]}
96
+ return {"error": ENGINE, "detail": f"{type(error).__name__}: {_detail(error)}"[:_DETAIL_LIMIT]}
97
+
98
+
99
+ class OutOfMemory:
100
+ """A request whose world code raised ``MemoryError``, or ``SystemError`` (what C code raises when an
101
+ allocation fails without setting one), noted in that request's own context where the raise becomes
102
+ its answer (:func:`failure_answer`, the host's request handler) and never read from the answer, so
103
+ no data a world serves can claim one. A worker's front end says so to its supervisor in ``HEADER``
104
+ on that answer, and sends no other header of that name; the supervisor recycles the worker only
105
+ when the kernel counts it near its cap."""
106
+
107
+ HEADER = "X-Gateway-Out-Of-Memory"
108
+ _noted: ContextVar[bool] = ContextVar("gateway_out_of_memory", default=False)
109
+
110
+ @classmethod
111
+ @contextmanager
112
+ def request(cls) -> Iterator[None]:
113
+ token = cls._noted.set(False)
114
+ try:
115
+ yield
116
+ finally:
117
+ cls._noted.reset(token)
118
+
119
+ @classmethod
120
+ def note(cls, error: BaseException) -> None:
121
+ if isinstance(error, (MemoryError, SystemError)):
122
+ cls._noted.set(True)
123
+
124
+ @classmethod
125
+ def noted(cls) -> bool:
126
+ return cls._noted.get()
127
+
128
+ @classmethod
129
+ def named(cls, header: str) -> bool:
130
+ return header.lower() == cls.HEADER.lower()
131
+
132
+ @classmethod
133
+ def in_headers(cls, headers: Iterable[tuple[str, str]]) -> bool:
134
+ return any(cls.named(name) for name, _ in headers)
93
135
 
94
136
 
95
137
  def engine_failure_body(handler: str, error: BaseException) -> dict:
@@ -99,6 +141,7 @@ def engine_failure_body(handler: str, error: BaseException) -> dict:
99
141
 
100
142
  def failure_answer(handler: str, error: Exception) -> tuple[int, dict]:
101
143
  """A handler's raise as its ``(status, body)``: a refusal, its author's bug or the engine's failure."""
144
+ OutOfMemory.note(error)
102
145
  if isinstance(error, Refusal):
103
146
  return refusal_answer(handler, error)
104
147
  if isinstance(error, HANDLER_BUG_ERRORS):
@@ -127,13 +170,24 @@ def unencodable(body: Any) -> Exception | None:
127
170
  return None
128
171
 
129
172
 
173
+ _HEADER_NAME = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
174
+ _LINE_BREAK = re.compile(r"[\r\n\x00]")
175
+
176
+
177
+ def header_line(name: str, value: str) -> bool:
178
+ """Whether ``name: value`` is one header line: a token for a name, no CR, LF or NUL in the value, so no
179
+ answer can add a line of its own (a forged ``OutOfMemory.HEADER`` included)."""
180
+ return bool(_HEADER_NAME.fullmatch(name)) and not _LINE_BREAK.search(value)
181
+
182
+
130
183
  def _unsendable_header(headers: dict[str, str] | None) -> Exception | None:
131
184
  try:
132
185
  for name, value in (headers or {}).items():
133
186
  f"{name}: {value}".encode("latin-1")
134
187
  except UnicodeEncodeError as error:
135
- return error
136
- return None
188
+ return ValueError(f"a header is not latin-1: {error}")
189
+ split = next((name for name, value in (headers or {}).items() if not header_line(name, value)), None)
190
+ return None if split is None else ValueError(f"header {split!r} is not one line: a token name, no CR, LF or NUL")
137
191
 
138
192
 
139
193
  def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None:
@@ -143,7 +197,7 @@ def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None
143
197
  return 400, handler_bug_body(handler, ValueError(f"an answer names a status from 200 to 599, not {status!r}"))
144
198
  header = _unsendable_header(headers)
145
199
  if header is not None:
146
- return 400, handler_bug_body(handler, header, "a header is not latin-1: ")
200
+ return 400, handler_bug_body(handler, header)
147
201
  error = unencodable(body)
148
202
  return None if error is None else (400, handler_bug_body(handler, error, "the answer is not JSON: "))
149
203
 
@@ -60,7 +60,7 @@ from .call_capture import JSON_TYPE, send_response
60
60
  from .call_redaction import CallRedaction
61
61
  from .credentials import BearerAuth
62
62
  from .door import PLATFORM_HEADER, Entry
63
- from .handler_errors import log_request_failure, request_failure_body
63
+ from .handler_errors import OutOfMemory, header_line, log_request_failure, request_failure_body
64
64
  from .host_browser import RelayNavigation, log_relay
65
65
  from .host_control import ControlPlane
66
66
  from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
@@ -100,6 +100,7 @@ from .workers import (
100
100
  DEFAULT_WORKER_BUDGET_MB,
101
101
  DEFAULT_WORKER_FILE_MB,
102
102
  DEFAULT_WORKER_FOOTPRINT_MB,
103
+ DEFAULT_WORKER_MEMORY_CEILING_MB,
103
104
  DEFAULT_WORKER_MEMORY_MB,
104
105
  WorkerPool,
105
106
  seal_process,
@@ -180,6 +181,20 @@ def make_host_handler(
180
181
  return {k.lower(): v for k, v in self.headers.items()}
181
182
 
182
183
  def _dispatch(self, method: str) -> None:
184
+ with OutOfMemory.request():
185
+ self._answer(method)
186
+
187
+ def send_header(self, keyword: str, value: str) -> None:
188
+ if OutOfMemory.named(keyword) or not header_line(keyword, str(value)):
189
+ return
190
+ super().send_header(keyword, value)
191
+
192
+ def end_headers(self) -> None:
193
+ if host.run_handlers and OutOfMemory.noted():
194
+ super().send_header(OutOfMemory.HEADER, "1")
195
+ super().end_headers()
196
+
197
+ def _answer(self, method: str) -> None:
183
198
  self._body: bytes | None = None
184
199
  url = urllib.parse.urlsplit(self.path)
185
200
  if url.path == "/healthz" and control.internal(self.path, self.headers):
@@ -203,6 +218,7 @@ def make_host_handler(
203
218
  self.close_connection = True
204
219
  return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
205
220
  except Exception as error:
221
+ OutOfMemory.note(error)
206
222
  log_request_failure(error, method, url.path)
207
223
  return self._reply(500, request_failure_body(error))
208
224
 
@@ -512,7 +528,8 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
512
528
  ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
513
529
  all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
514
530
  its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
515
- ``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
531
+ ``WORLDS_WORKER_MEMORY_CEILING_MB`` the most address space a worker measured to need more is given (unset,
532
+ the cap), ``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
516
533
  ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
517
534
  host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
518
535
  ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
@@ -540,6 +557,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
540
557
  lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
541
558
  workers = None
542
559
  footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
560
+ ceiling = os.environ.get("WORLDS_WORKER_MEMORY_CEILING_MB")
543
561
  if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
544
562
  workers = WorkerPool(
545
563
  Path(cache_root),
@@ -548,6 +566,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
548
566
  budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
549
567
  footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
550
568
  file_mb=int(os.environ.get("WORLDS_WORKER_FILE_MB") or DEFAULT_WORKER_FILE_MB),
569
+ ceiling_mb=int(ceiling) if ceiling else DEFAULT_WORKER_MEMORY_CEILING_MB,
551
570
  )
552
571
  owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
553
572
  world_host = WorldHost(
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any
16
16
 
17
17
  from .handler_errors import carried
18
18
  from .host_browser import RelayNavigation
19
- from .host_requests import BrowserRelay
19
+ from .host_requests import BrowserRelay, parse_memory_peak, parse_world_state
20
20
  from .host_surfaces import public_host
21
21
  from .host_workers import WORKER_EXITED, refusal
22
22
  from .validation import ContractError, fail
@@ -148,6 +148,8 @@ class ControlPlane:
148
148
  auth=body.get("auth"),
149
149
  ledger_source=body.get("ledgerSource"),
150
150
  parameters=body.get("parameters"),
151
+ world_state=parse_world_state(body.get("worldState")),
152
+ memory_peak_mb=parse_memory_peak(body.get("memoryPeakMb")),
151
153
  )
152
154
 
153
155
  def _swap_browser(self, session_id: str, browser: Any) -> dict:
@@ -4,6 +4,7 @@ platform what the host is and holds. A step that fails is logged and the next on
4
4
 
5
5
  from __future__ import annotations
6
6
 
7
+ import http.client
7
8
  import json
8
9
  import threading
9
10
  import urllib.error
@@ -18,6 +19,8 @@ if TYPE_CHECKING:
18
19
 
19
20
  LEASE_SWEEP_S = 30.0
20
21
  HEARTBEAT_S = 10.0
22
+ #: The most a heartbeat reads of the platform's answer.
23
+ ANSWER_LIMIT_BYTES = 64 << 10
21
24
 
22
25
 
23
26
  def swept(step: Callable[[], list[str]], done: str) -> list[str]:
@@ -49,7 +52,8 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
49
52
  """Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
50
53
  The platform's registry is what places new sessions: a host it has not
51
54
  heard from lately gets none. Returns whether the platform took it."""
52
- body = json.dumps(world_host.stats()).encode("utf-8")
55
+ stats = world_host.stats()
56
+ body = json.dumps(stats).encode("utf-8")
53
57
  req = urllib.request.Request(
54
58
  f"{platform_url.rstrip('/')}/api/public/world-hosts",
55
59
  data=body,
@@ -58,10 +62,23 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
58
62
  )
59
63
  try:
60
64
  with urllib.request.urlopen(req, timeout=timeout_s) as r:
61
- return 200 <= r.status < 300
62
- except (urllib.error.URLError, OSError, ValueError) as error:
65
+ taken = 200 <= r.status < 300
66
+ peaks_taken = taken and _peaks_taken(r.read(ANSWER_LIMIT_BYTES))
67
+ except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
63
68
  log_line(f"heartbeat failed: {error}")
64
69
  return False
70
+ if peaks_taken:
71
+ world_host.reported(stats)
72
+ return taken
73
+
74
+
75
+ def _peaks_taken(answer: bytes) -> bool:
76
+ """Whether the platform says it kept the beat's worker peaks: one that predates them takes the beat
77
+ and ignores them, so they stay pending until a platform that keeps them answers."""
78
+ try:
79
+ return json.loads(answer).get("peaksTaken") is True
80
+ except (ValueError, AttributeError):
81
+ return False
65
82
 
66
83
 
67
84
  class Heartbeat:
@@ -90,8 +107,14 @@ class Heartbeat:
90
107
  self._due.set()
91
108
 
92
109
  def _beat(self) -> None:
93
- heartbeat_once(self.world_host, self.platform_url, self.secret)
110
+ self._once()
94
111
  while not self._stopped.is_set():
95
112
  self._due.wait(self.interval_s)
96
113
  self._due.clear()
114
+ self._once()
115
+
116
+ def _once(self) -> None:
117
+ try:
97
118
  heartbeat_once(self.world_host, self.platform_url, self.secret)
119
+ except Exception as error: # noqa: BLE001 - one failed beat must not stop the beats after it
120
+ log_line(f"heartbeat failed: {type(error).__name__}")
@@ -0,0 +1,69 @@
1
+ """The prometheus exposition of a World Host's :meth:`WorldHost.stats`. ``worlds_host_sessions`` is the
2
+ number the fleet scales on (sessions per host, not cpu)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from .host_handoff import OUTCOMES
7
+ from .host_workers import WORKERS_POISONED
8
+ from .worker_watch import WORKERS_NEAR_CAP
9
+
10
+
11
+ def _largest_share(workers: list[dict]) -> list[str]:
12
+ """The most of its own cap any worker holds (``VmSize`` over ``RLIMIT_AS``), no sample when none was
13
+ measured. A total: world code on the pod can read ``/metrics``, so no line here names a version."""
14
+ shares = [w["sizeMb"] / w["capMb"] for w in workers if w.get("sizeMb") is not None and w.get("capMb")]
15
+ sample = [f"worlds_host_worker_cap_share_max {max(shares):.3f}"] if shares else []
16
+ return ["# TYPE worlds_host_worker_cap_share_max gauge", *sample]
17
+
18
+
19
+ def exposition(stats: dict) -> str:
20
+ counters: dict[str, int] = stats["counters"]
21
+ lines = [
22
+ "# TYPE worlds_host_sessions gauge",
23
+ f"worlds_host_sessions {stats['sessions']}",
24
+ "# TYPE worlds_host_capacity gauge",
25
+ f"worlds_host_capacity {stats['capacity']}",
26
+ "# TYPE worlds_host_draining gauge",
27
+ f"worlds_host_draining {int(stats['draining'])}",
28
+ "# TYPE worlds_host_workers gauge",
29
+ f"worlds_host_workers {stats['workers']}",
30
+ "# TYPE worlds_host_worker_memory_mb gauge",
31
+ f"worlds_host_worker_memory_mb {stats['workerMemoryMb']}",
32
+ "# TYPE worlds_host_versions_cached gauge",
33
+ f"worlds_host_versions_cached {len(stats['versions'])}",
34
+ "# TYPE worlds_host_uptime_seconds gauge",
35
+ f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
36
+ "# TYPE worlds_host_sessions_opened_total counter",
37
+ f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
38
+ "# TYPE worlds_host_open_failures_total counter",
39
+ f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
40
+ "# TYPE worlds_host_session_seconds_total counter",
41
+ f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
42
+ "# TYPE worlds_host_exports_total counter",
43
+ f"worlds_host_exports_total {counters.get('exports', 0)}",
44
+ "# TYPE worlds_host_exports_uploaded_total counter",
45
+ f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
46
+ "# TYPE worlds_host_sessions_closed_total counter",
47
+ ]
48
+ for reason in ("closed", "expired", "lost", "replaced", "moved"):
49
+ lines.append(f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}')
50
+ lines.append("# TYPE worlds_host_handoffs_total counter")
51
+ for outcome in OUTCOMES:
52
+ lines.append(f'worlds_host_handoffs_total{{outcome="{outcome}"}} {counters.get(f"handoffs.{outcome}", 0)}')
53
+ workers = stats.get("workerMemory") or []
54
+ lines += _largest_share(workers)
55
+ lines += [
56
+ "# TYPE worlds_host_workers_near_cap gauge",
57
+ f"worlds_host_workers_near_cap {sum(1 for w in workers if w.get('nearCap'))}",
58
+ ]
59
+ recoveries = ("workers_replaced", "workers_recycled", WORKERS_POISONED, WORKERS_NEAR_CAP, "sessions_recovered")
60
+ for name in (*recoveries, "sessions_unrecovered"):
61
+ lines += [f"# TYPE worlds_host_{name}_total counter", f"worlds_host_{name}_total {counters.get(name, 0)}"]
62
+ lines.append("# TYPE worlds_host_requests_total counter")
63
+ for surface in ("api", "ui", "browser", "proxied"):
64
+ lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
65
+ lines += [
66
+ "# TYPE worlds_host_platform_bearer_requests_total counter",
67
+ f"worlds_host_platform_bearer_requests_total {counters.get('requests.platform_bearer', 0)}",
68
+ ]
69
+ return "\n".join(lines) + "\n"