@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -93,6 +93,56 @@ def pointer(part: Any) -> str:
93
93
  return str(part).replace("~", "~0").replace("/", "~1")
94
94
 
95
95
 
96
+ LOCAL_DEFS = "#/$defs/"
97
+ COMPOSITIONS = ("allOf", "anyOf", "oneOf")
98
+
99
+
100
+ class InlineBudgetExceeded(Exception):
101
+ """Inlining every reference produced more schema nodes than a publishable contract holds."""
102
+
103
+
104
+ def local_reference(key: str, definition: dict) -> dict:
105
+ """A compiled reference to a shared definition, carrying the definition's type so a
106
+ reader that looks one level deep (columns, the generated ui) still sees it."""
107
+ typed = {"type": definition["type"]} if "type" in definition else {}
108
+ return {"$ref": LOCAL_DEFS + pointer(key), **typed}
109
+
110
+
111
+ def local_key(ref: Any) -> str | None:
112
+ if not isinstance(ref, str) or not ref.startswith(LOCAL_DEFS):
113
+ return None
114
+ return ref[len(LOCAL_DEFS) :].replace("~1", "/").replace("~0", "~")
115
+
116
+
117
+ def local_definition(defs: dict, ref: Any, location: str) -> dict:
118
+ key = local_key(ref)
119
+ if key not in defs:
120
+ fail("reference", f"Unresolved reference {ref}", location)
121
+ return defs[key]
122
+
123
+
124
+ def subschemas(schema: dict) -> list[dict]:
125
+ children = list(schema.get("properties", {}).values())
126
+ children += [schema[key] for key in ("items", "additionalProperties") if isinstance(schema.get(key), dict)]
127
+ return children + [branch for key in COMPOSITIONS for branch in schema.get(key, [])]
128
+
129
+
130
+ def structural(schema: dict) -> bool:
131
+ """Whether a compiled schema can hold an object or array: such a definition is worth
132
+ sharing; a scalar one (an enum, a formatted string) is inlined where it is used."""
133
+ types = schema.get("type", [])
134
+ types = {types} if isinstance(types, str) else set(types)
135
+ branches = [branch for key in COMPOSITIONS for branch in schema.get(key, [])]
136
+ return bool(types & {"object", "array"}) or "$ref" in schema or any(structural(b) for b in branches)
137
+
138
+
139
+ def local_references(schema: dict) -> set[str]:
140
+ found = {local_key(schema["$ref"])} if "$ref" in schema else set()
141
+ for child in subschemas(schema):
142
+ found |= local_references(child)
143
+ return found
144
+
145
+
96
146
  def json_value(value: Any, path: str = "") -> None:
97
147
  """Require JSON-native values and the explicitly supported numeric range."""
98
148
  if value is None or type(value) is bool:
@@ -221,15 +271,18 @@ def nonempty(value: Any, location: str) -> None:
221
271
 
222
272
  #: A connector operation name: snake_case, at most this long. The bound is shared with
223
273
  #: the TypeScript ``route add`` (``MAX_OPERATION_NAME`` in scaffoldRoutes.ts) and the
224
- #: OpenAPI importer, which truncates an ``operationId`` to fit; a 300-character name
225
- #: used to compile (zero-context round 2, W3).
274
+ #: OpenAPI importer, which truncates an ``operationId`` to fit.
226
275
  MAX_OPERATION_NAME = 64
227
276
  #: An entity name: at most this long, so ``data-import:<entity>:<identity>`` source
228
- #: records and SQLite table names stay inside the 256-character record limit. Shared
277
+ #: records and table names stay inside the 256-character record limit. Shared
229
278
  #: with the TypeScript ``schema entity add`` (``MAX_ENTITY_NAME`` in scaffold.ts) and
230
279
  #: the OpenAPI importer, which truncates a schema name to fit.
231
280
  MAX_ENTITY_NAME = 64
232
281
  OPERATION_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
282
+ #: A vendor-named field (an entity property, a redacted field) as the vendor spells it:
283
+ #: hyphens kept (``reference-count``), never a dot or slash, which address nested fields.
284
+ #: The TypeScript ``schema entity add`` pins the same rule (``PROPERTY_NAME`` in scaffoldJson.ts).
285
+ FIELD_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
233
286
 
234
287
 
235
288
  def parse_toml(text: str) -> dict:
@@ -265,8 +318,9 @@ def equal(left: Any, right: Any) -> bool:
265
318
  return left == right
266
319
 
267
320
 
268
- def validate(schema: dict, value: Any, *, location: str = "", path: str = "") -> None:
321
+ def validate(schema: dict, value: Any, *, location: str = "", path: str = "", defs: dict | None = None) -> None:
269
322
  json_value(value, path)
323
+ defs = schema.get("$defs", {}) if defs is None else defs
270
324
 
271
325
  def error(code: str, message: str, keyword: str = "", **context: Any) -> NoReturn:
272
326
  fail(code, message, f"{location}/{keyword}" if keyword else location, path=path, **context)
@@ -289,17 +343,20 @@ def validate(schema: dict, value: Any, *, location: str = "", path: str = "") ->
289
343
  )
290
344
  if types and not matches:
291
345
  error("type", f"Expected {' or '.join(types)}", "type")
346
+ if "$ref" in schema:
347
+ target = local_definition(defs, schema["$ref"], f"{location}/$ref")
348
+ validate(target, value, location=f"{location}/$ref", path=path, defs=defs)
292
349
  if "enum" in schema and not any(equal(value, option) for option in schema["enum"]):
293
350
  error("enum", "Value is outside the declared enum", "enum")
294
351
  if "const" in schema and not equal(value, schema["const"]):
295
352
  error("const", "Value differs from the declared constant", "const")
296
- for keyword in ("allOf", "anyOf", "oneOf"):
353
+ for keyword in COMPOSITIONS:
297
354
  if keyword not in schema:
298
355
  continue
299
356
  matches_count = 0
300
357
  for index, branch in enumerate(schema[keyword]):
301
358
  try:
302
- validate(branch, value, location=f"{location}/{keyword}/{index}", path=path)
359
+ validate(branch, value, location=f"{location}/{keyword}/{index}", path=path, defs=defs)
303
360
  matches_count += 1
304
361
  except ContractError:
305
362
  pass
@@ -318,6 +375,7 @@ def validate(schema: dict, value: Any, *, location: str = "", path: str = "") ->
318
375
  item,
319
376
  location=f"{location}/properties/{pointer(name)}",
320
377
  path=f"{path}/{pointer(name)}",
378
+ defs=defs,
321
379
  )
322
380
  elif schema.get("additionalProperties") is False:
323
381
  error(
@@ -327,13 +385,12 @@ def validate(schema: dict, value: Any, *, location: str = "", path: str = "") ->
327
385
  property=name,
328
386
  )
329
387
  elif isinstance(schema.get("additionalProperties"), dict):
330
- # A schema-valued additionalProperties types a map: every
331
- # undeclared member holds a value of that schema.
332
388
  validate(
333
389
  schema["additionalProperties"],
334
390
  item,
335
391
  location=f"{location}/additionalProperties",
336
392
  path=f"{path}/{pointer(name)}",
393
+ defs=defs,
337
394
  )
338
395
  if type(value) is list:
339
396
  for keyword, invalid in (
@@ -353,6 +410,7 @@ def validate(schema: dict, value: Any, *, location: str = "", path: str = "") ->
353
410
  item,
354
411
  location=f"{location}/items",
355
412
  path=f"{path}/{index}",
413
+ defs=defs,
356
414
  )
357
415
  if type(value) is str:
358
416
  if len(value) < schema.get("minLength", 0) or len(value) > schema.get("maxLength", math.inf):
@@ -394,14 +452,25 @@ def validate(schema: dict, value: Any, *, location: str = "", path: str = "") ->
394
452
 
395
453
 
396
454
  class Resolver:
397
- """Resolve only relative files under schema/ and JSON pointers into $defs."""
455
+ """Resolve only relative files under schema/ and JSON pointers into $defs.
456
+
457
+ Inlined (the default) every reference is replaced by its target, and a nested
458
+ occurrence of a type inside itself keeps only its bare type. Shared, a reference
459
+ below a root to a structural definition becomes a local ``$ref`` and the definition
460
+ is resolved once, so the output grows with the input rather than with the number of
461
+ paths through it. ``budget`` caps how many schema nodes inlining may produce."""
398
462
 
399
- def __init__(self, bundle: Path):
463
+ def __init__(self, bundle: Path, *, shared: bool = False, budget: int | None = None):
400
464
  self.root = (bundle / "schema").resolve()
401
465
  self.documents: dict[str, dict] = {}
402
466
  self.identities: dict[str, str] = {}
403
467
  self.checked: set[str] = set()
404
- self.checking: set[str] = set()
468
+ self.shared = shared
469
+ self.budget = budget
470
+ self.definitions: dict[str, dict] = {}
471
+ self.defining: set[str] = set()
472
+ self.projections: dict[str, dict] = {}
473
+ self.locations: dict[str, str] = {}
405
474
 
406
475
  def load(self, name: str) -> dict:
407
476
  path = self.root / name
@@ -460,7 +529,9 @@ class Resolver:
460
529
  stack: tuple[str, ...] = (),
461
530
  *,
462
531
  root: bool = False,
532
+ nested: bool = False,
463
533
  ) -> dict:
534
+ self.spend()
464
535
  if len(stack) > 80:
465
536
  fail("reference", "Reference nesting exceeds 80", location)
466
537
  if type(schema) is not dict:
@@ -493,46 +564,32 @@ class Resolver:
493
564
  fail("schema", "examples must be an array", location)
494
565
  result = {k: v for k, v in schema.items() if k not in {"$ref", "$defs", "$schema", "$id", "x-gateway"}}
495
566
  if "$ref" in schema:
496
- if set(schema) - DOC_KEYS - {"$ref"}:
497
- fail(
498
- "unsupported",
499
- "$ref siblings may only be documentation annotations; use allOf for assertions",
500
- location,
501
- )
502
- target, target_name, target_location = self.reference(schema["$ref"], name, location)
503
- if target_location in stack:
504
- # Vendor models nest a type in itself (Brex's ExpandableExpense
505
- # holds an ExpandableExpense). Inlining would never end, so the
506
- # nested occurrence keeps only the target's bare type: it is
507
- # validated as that type and stored as JSON, never projected.
508
- return {**self.recursive(target, target_name, target_location, location), **result}
509
- result = {
510
- **self.resolve(target, target_name, target_location, (*stack, target_location)),
511
- **result,
512
- }
567
+ return self._reference(schema, result, name, location, stack, nested=nested)
513
568
  for key in ("$defs", "properties"):
514
569
  if key not in schema:
515
570
  continue
516
571
  if type(schema[key]) is not dict:
517
572
  fail("schema", f"{key} must be an object", f"{location}/{key}")
518
573
  children = {
519
- k: self.resolve(v, name, f"{location}/{key}/{pointer(k)}", stack)
574
+ k: self.resolve(v, name, f"{location}/{key}/{pointer(k)}", stack, nested=True)
520
575
  for k, v in sorted(schema[key].items())
521
576
  }
522
577
  if key == "properties":
523
578
  result[key] = children
524
579
  if "items" in schema:
525
- result["items"] = self.resolve(schema["items"], name, f"{location}/items", stack)
580
+ result["items"] = self.resolve(schema["items"], name, f"{location}/items", stack, nested=True)
526
581
  if isinstance(schema.get("additionalProperties"), dict):
527
- # A map of one value type (Brex's `Record<string, string>` shapes).
528
582
  result["additionalProperties"] = self.resolve(
529
- schema["additionalProperties"], name, f"{location}/additionalProperties", stack
583
+ schema["additionalProperties"], name, f"{location}/additionalProperties", stack, nested=True
530
584
  )
531
585
  for key in ("allOf", "anyOf", "oneOf"):
532
586
  if key in schema:
533
587
  if not isinstance(schema[key], list) or not schema[key]:
534
588
  fail("schema", f"{key} needs a nonempty schema array", location)
535
- result[key] = [self.resolve(s, name, f"{location}/{key}/{i}", stack) for i, s in enumerate(schema[key])]
589
+ result[key] = [
590
+ self.resolve(s, name, f"{location}/{key}/{i}", stack, nested=True)
591
+ for i, s in enumerate(schema[key])
592
+ ]
536
593
  types = result.get("type", [])
537
594
  types = [types] if isinstance(types, str) else types
538
595
  if (
@@ -554,9 +611,7 @@ class Resolver:
554
611
  if isinstance(result.get("type"), list):
555
612
  result["type"] = sorted(types)
556
613
  if "format" in result and "string" not in types and set(types) & {"integer", "number"}:
557
- # OpenAPI documents annotate numbers with `int64`, `int32`, `double`
558
- # (Brex). Those are storage hints, not assertions: drop them rather
559
- # than refuse the document. String formats stay validated below.
614
+ # OpenAPI's int64 / int32 / double on a number are storage hints, not assertions.
560
615
  del result["format"]
561
616
  applicability = {
562
617
  "properties": {"object"},
@@ -619,10 +674,78 @@ class Resolver:
619
674
  or any(equal(v, p) for i, v in enumerate(values) for p in values[:i])
620
675
  ):
621
676
  fail("schema", "enum must contain distinct values", location)
622
- if "default" in result:
623
- validate(result, result["default"], location=f"{location}/default")
677
+ self._check_default(result, location)
624
678
  return result
625
679
 
680
+ def _reference(
681
+ self, schema: dict, annotations: dict, name: str, location: str, stack: tuple[str, ...], *, nested: bool
682
+ ) -> dict:
683
+ """A ``$ref`` beside documentation annotations alone. Shared, one below a root is a
684
+ local reference to its definition. A target already on this path (a vendor type
685
+ nested in itself, which would inline forever) keeps only its bare type: validated
686
+ as that type and stored as JSON, never projected. Otherwise the target is inlined."""
687
+ if set(schema) - DOC_KEYS - {"$ref"}:
688
+ fail(
689
+ "unsupported",
690
+ "$ref siblings may only be documentation annotations; use allOf for assertions",
691
+ location,
692
+ )
693
+ target, target_name, target_location = self.reference(schema["$ref"], name, location)
694
+ if self.shared and nested:
695
+ return {**self.definition(target, target_name, target_location, location), **annotations}
696
+ if target_location in stack:
697
+ return {**self.recursive(target, target_name, target_location, location), **annotations}
698
+ inlined = {**self.resolve(target, target_name, target_location, (*stack, target_location)), **annotations}
699
+ self._check_default(inlined, location)
700
+ return inlined
701
+
702
+ def _check_default(self, schema: dict, location: str) -> None:
703
+ """A ``default`` must be a value of its schema. A definition still being resolved
704
+ (a type nested in itself) is read as its bare type, as inlining reads it."""
705
+ if "default" not in schema:
706
+ return
707
+ defs = {**self.projections, **self.definitions}
708
+ validate(schema, schema["default"], location=f"{location}/default", defs=defs)
709
+
710
+ def spend(self) -> None:
711
+ if self.shared or self.budget is None:
712
+ return
713
+ self.budget -= 1
714
+ if self.budget < 0:
715
+ raise InlineBudgetExceeded
716
+
717
+ def standalone(self, schema: Any, name: str, location: str) -> dict:
718
+ """A root schema (an entity, a tool input or output) with the shared definitions it
719
+ reaches under its own ``$defs``, so it validates and serves on its own."""
720
+ body = self.resolve(schema, name, location)
721
+ defs: dict[str, dict] = {}
722
+ pending = local_references(body)
723
+ while pending:
724
+ key = pending.pop()
725
+ defs[key] = self.definitions[key]
726
+ pending |= local_references(defs[key]) - set(defs)
727
+ return {**body, "$defs": dict(sorted(defs.items()))} if defs else body
728
+
729
+ def definition(self, target: Any, name: str, target_location: str, location: str) -> dict:
730
+ key = self.key(target_location)
731
+ if key in self.defining:
732
+ projection = self.recursive(target, name, target_location, location)
733
+ self.projections[key] = projection
734
+ return local_reference(key, projection) if structural(projection) else projection
735
+ if key not in self.definitions:
736
+ self.defining.add(key)
737
+ self.definitions[key] = self.resolve(target, name, target_location, (target_location,))
738
+ self.defining.discard(key)
739
+ body = self.definitions[key]
740
+ return local_reference(key, body) if structural(body) else dict(body)
741
+
742
+ def key(self, target_location: str) -> str:
743
+ filename, fragment = target_location.split("#", 1)
744
+ key = Path(filename).stem + fragment.replace("/$defs/", ".")
745
+ if self.locations.setdefault(key, target_location) != target_location:
746
+ fail("reference", f"Definitions {self.locations[key]} and {target_location} share the name {key}", key)
747
+ return key
748
+
626
749
  def recursive(self, target: Any, name: str, target_location: str, location: str) -> dict:
627
750
  """The bare-type projection of a reference target already being resolved.
628
751
 
@@ -652,8 +775,11 @@ class Resolver:
652
775
  }
653
776
 
654
777
  def check_documents(self) -> None:
655
- # Resolving references can discover more files. Check all definitions,
656
- # including unused ones, so unsupported assertions never hide there.
778
+ """Check every document whole, unused definitions included, so an unsupported
779
+ assertion never hides in one; resolving may load more documents, checked in turn.
780
+ The check runs in this resolver's own form: an inlined build is checked inlined,
781
+ so a world that compiled inlined before compiles the same (the shared form names
782
+ definitions and reads defaults through them, which inlining never did)."""
657
783
  while set(self.documents) - self.checked:
658
784
  name = sorted(set(self.documents) - self.checked)[0]
659
785
  self.resolve(self.documents[name], name, f"{name}#", root=True)
@@ -9,7 +9,7 @@ it. The worker is the same :class:`host.WorldHost` in a smaller room:
9
9
  * its own process, so a handler that loops, leaks or crashes takes down that
10
10
  version's sessions on this replica and nothing else;
11
11
  * rlimits on address space, open files and file size, set before any world
12
- code is imported;
12
+ code is imported (:class:`AddressSpaceLimit` says how macOS differs);
13
13
  * a scrubbed environment: the control-plane secret, cloud credentials and the
14
14
  rest of the supervisor's environment are not inherited. The worker gets a
15
15
  secret of its own, good for its socket alone;
@@ -17,10 +17,11 @@ it. The worker is the same :class:`host.WorldHost` in a smaller room:
17
17
  a process even when they share a host;
18
18
  * a socket only the supervisor is answered on. Workers run as the host's user,
19
19
  so file modes cannot keep one worker's handler from opening a sibling's
20
- socket path: the worker asks the kernel who connected (``SO_PEERCRED``) and
21
- serves the supervisor's pid alone. The listening socket is the supervisor's,
22
- bound before the worker starts and handed down as a file descriptor, so a
23
- worker never binds a path of its own;
20
+ socket path: the worker asks the kernel who connected (``SO_PEERCRED``, on
21
+ macOS ``LOCAL_PEERPID``) and serves the supervisor's pid alone. The listening
22
+ socket is the supervisor's, bound before the worker starts in a short private
23
+ directory and handed down as a file descriptor, so a worker never binds a
24
+ path of its own;
24
25
  * a worker nobody can read: it marks itself non-dumpable, which closes
25
26
  ``/proc/<pid>/environ``, ``/proc/<pid>/mem`` and ``ptrace`` to every process
26
27
  that is not root, siblings under the same uid included. Its secret and its
@@ -35,6 +36,7 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
35
36
  from __future__ import annotations
36
37
 
37
38
  import ctypes
39
+ import hashlib
38
40
  import http.client
39
41
  import os
40
42
  import resource
@@ -44,12 +46,14 @@ import socket
44
46
  import struct
45
47
  import subprocess
46
48
  import sys
49
+ import tempfile
47
50
  import threading
48
51
  import time
49
52
  from collections.abc import Mapping
50
53
  from dataclasses import dataclass, field
51
54
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
52
55
  from pathlib import Path
56
+ from typing import Protocol
53
57
 
54
58
  from .grading import VerifierSandbox
55
59
 
@@ -63,17 +67,20 @@ DEFAULT_WORKER_MEMORY_MB = 2048
63
67
  DEFAULT_WORKER_BUDGET_MB = 4096
64
68
  #: What one worker is counted at against the budget, in MiB, unless
65
69
  #: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise. Unset, a worker is counted at
66
- #: its whole address-space cap — two of 2 GiB inside a 6 GiB container — which
67
- #: is the safe reading and a 50x over-reservation: a measured worker holding
68
- #: sessions is ~40 MiB resident (prod, 2026-09-17). A fleet that must host many
69
- #: worlds at once sets this to the footprint it has measured, keeps the
70
- #: address-space cap as the rail each worker cannot cross, and sizes the
71
- #: container for footprint x workers plus the supervisor's share.
70
+ #: its whole address-space cap: the safe reading, and a 50x over-reservation of
71
+ #: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
72
+ #: many worlds at once sets the footprint it measured, keeps the address-space
73
+ #: cap as the rail each worker cannot cross, and sizes the container for
74
+ #: footprint x workers plus the supervisor's share.
72
75
  DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
73
76
  #: Largest file a worker may write (its session databases), in MiB.
74
77
  DEFAULT_WORKER_FILE_MB = 4096
75
78
  #: Open files per worker: one per live session plus the sockets serving them.
76
79
  DEFAULT_WORKER_NOFILE = 4096
80
+ FLOOR_PRECISION_BYTES = 1 << 20
81
+ #: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
82
+ #: spells it so, Linux spells it -1.
83
+ MAX_RLIMIT = 2**63 - 1
77
84
  #: How long a worker with no sessions is kept before it is stopped.
78
85
  DEFAULT_WORKER_IDLE_S = 300.0
79
86
  #: How long the supervisor waits for a fresh worker to answer on its socket.
@@ -98,11 +105,6 @@ class WorkerCapacity(Exception):
98
105
  worker for another version can start here."""
99
106
 
100
107
 
101
- # --------------------------------------------------------------------------- #
102
- # HTTP over a unix socket: the only way into a worker.
103
- # --------------------------------------------------------------------------- #
104
-
105
-
106
108
  class UnixHTTPConnection(http.client.HTTPConnection):
107
109
  def __init__(self, path: str | Path, timeout: float | None = None):
108
110
  super().__init__("worker", timeout=timeout)
@@ -128,18 +130,40 @@ def bind_worker_socket(path: str | Path) -> socket.socket:
128
130
  return sock
129
131
 
130
132
 
131
- def peer_pid(conn: socket.socket) -> int | None:
132
- """The pid of whoever connected, as the kernel reports it (``SO_PEERCRED``:
133
- ``struct ucred {pid, uid, gid}``); None where the platform has no such
134
- thing."""
135
- option = getattr(socket, "SO_PEERCRED", None)
136
- if option is None:
137
- return None
138
- raw = conn.getsockopt(socket.SOL_SOCKET, option, struct.calcsize("3i"))
133
+ #: macOS's ``SOL_LOCAL`` and ``LOCAL_PEERPID`` from <sys/un.h>; CPython exports neither.
134
+ _SOL_LOCAL, _LOCAL_PEERPID = 0, 0x002
135
+
136
+
137
+ def _linux_peer_pid(conn: socket.socket) -> int:
138
+ raw = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))
139
139
  pid, _uid, _gid = struct.unpack("3i", raw)
140
140
  return pid
141
141
 
142
142
 
143
+ def _darwin_peer_pid(conn: socket.socket) -> int:
144
+ raw = conn.getsockopt(_SOL_LOCAL, _LOCAL_PEERPID, struct.calcsize("i"))
145
+ return struct.unpack("i", raw)[0]
146
+
147
+
148
+ _PEER_PID = {"linux": _linux_peer_pid, "darwin": _darwin_peer_pid}
149
+
150
+
151
+ def peer_pid(conn: socket.socket) -> int | None:
152
+ """The pid of whoever connected, as the kernel reports it: Linux's
153
+ ``SO_PEERCRED`` (``struct ucred {pid, uid, gid}``), macOS's
154
+ ``LOCAL_PEERPID``. None where the platform has no such thing, or where the
155
+ kernel cannot name the peer (macOS raises ``ENOTCONN`` once the client has
156
+ hung up): a worker serving its supervisor alone refuses both, and keeps
157
+ serving, since ``socketserver`` asks outside its own error handling."""
158
+ read = _PEER_PID.get(sys.platform)
159
+ if read is None:
160
+ return None
161
+ try:
162
+ return read(conn)
163
+ except OSError:
164
+ return None
165
+
166
+
143
167
  class UnixHTTPServer(ThreadingHTTPServer):
144
168
  """The worker's server: a ``ThreadingHTTPServer`` over the listening unix
145
169
  socket the supervisor handed it (an inherited descriptor), or one it binds
@@ -195,37 +219,100 @@ def seal_process() -> bool:
195
219
  return False
196
220
 
197
221
 
222
+ class Kernel(Protocol):
223
+ """Where a process's limits are read and set: the ``resource`` module."""
224
+
225
+ RLIM_INFINITY: int
226
+
227
+ def getrlimit(self, name: int, /) -> tuple[int, int]: ...
228
+
229
+ def setrlimit(self, name: int, limits: tuple[int, int], /) -> None: ...
230
+
231
+
232
+ @dataclass(frozen=True)
233
+ class ProcessLimit:
234
+ """One rlimit on this process, soft and hard alike. A limit the kernel
235
+ holds lower than asked is lowered to its hard limit, never raised; one it
236
+ refuses fails the start."""
237
+
238
+ name: int
239
+ wanted: int
240
+ label: str
241
+ kernel: Kernel = resource
242
+
243
+ def apply(self) -> int:
244
+ value = self.within_hard()
245
+ self.kernel.setrlimit(self.name, (value, value))
246
+ return value
247
+
248
+ def hard(self) -> int:
249
+ return self.kernel.getrlimit(self.name)[1]
250
+
251
+ def ceiling(self) -> int:
252
+ """The hard limit as a number: ``RLIM_INFINITY`` reads as :data:`MAX_RLIMIT`."""
253
+ hard = self.hard()
254
+ return MAX_RLIMIT if hard == self.kernel.RLIM_INFINITY else hard
255
+
256
+ def within_hard(self) -> int:
257
+ return min(self.wanted, self.ceiling())
258
+
259
+
260
+ class AddressSpaceLimit(ProcessLimit):
261
+ """The memory cap. Linux takes any cap up to the hard limit, so there the
262
+ budget is the cap exactly. macOS refuses a cap below what the process
263
+ already maps (about 392 GiB on Apple Silicon, mostly the shared region's
264
+ reservation), so there the budget is granted on top of that floor: it still
265
+ bounds growth, and a kernel that takes no cap at all fails the start."""
266
+
267
+ def apply(self) -> int:
268
+ try:
269
+ return super().apply()
270
+ except ValueError:
271
+ return self._above_floor()
272
+
273
+ def _above_floor(self) -> int:
274
+ ceiling = self.ceiling()
275
+ floor = self._lowest_taken(ceiling)
276
+ if floor >= ceiling:
277
+ raise ValueError(f"the kernel refuses every {self.label} cap under its hard limit")
278
+ value = min(floor + self.wanted, ceiling)
279
+ self.kernel.setrlimit(self.name, (value, value))
280
+ return value
281
+
282
+ def _lowest_taken(self, ceiling: int) -> int:
283
+ """The lowest cap the kernel takes, to within FLOOR_PRECISION_BYTES.
284
+ Each probe moves the soft limit alone and is undone at once: a lowered
285
+ hard limit cannot be raised again, and a process left near its floor
286
+ fails its next allocation."""
287
+ restore = self.kernel.getrlimit(self.name)
288
+ refused, taken = self.within_hard(), ceiling
289
+ while taken - refused > FLOOR_PRECISION_BYTES:
290
+ probe = (refused + taken) // 2
291
+ try:
292
+ self.kernel.setrlimit(self.name, (probe, restore[1]))
293
+ except ValueError:
294
+ refused = probe
295
+ continue
296
+ self.kernel.setrlimit(self.name, restore)
297
+ taken = probe
298
+ return taken
299
+
300
+
198
301
  def apply_limits(
199
302
  *,
200
303
  memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
201
304
  file_mb: int = DEFAULT_WORKER_FILE_MB,
202
305
  nofile: int = DEFAULT_WORKER_NOFILE,
203
306
  ) -> dict[str, int]:
204
- """Cap this process before it imports world code. A limit the kernel will
205
- not grant (a hard limit lower than asked) is lowered to what it grants
206
- rather than refused: the worker still runs, just tighter. Returns the
207
- limits as set, so the worker can report them."""
208
- applied: dict[str, int] = {}
209
- for name, wanted in (
210
- (resource.RLIMIT_AS, memory_mb << 20),
211
- (resource.RLIMIT_FSIZE, file_mb << 20),
212
- (resource.RLIMIT_NOFILE, nofile),
213
- (resource.RLIMIT_CORE, 0),
214
- ):
215
- _, hard = resource.getrlimit(name)
216
- value = wanted if hard == resource.RLIM_INFINITY else min(wanted, hard)
217
- resource.setrlimit(name, (value, value))
218
- applied[_limit_name(name)] = value
219
- return applied
220
-
221
-
222
- def _limit_name(limit: int) -> str:
223
- return {
224
- resource.RLIMIT_AS: "memory_bytes",
225
- resource.RLIMIT_FSIZE: "file_bytes",
226
- resource.RLIMIT_NOFILE: "open_files",
227
- resource.RLIMIT_CORE: "core_bytes",
228
- }[limit]
307
+ """Cap this process before it imports world code. Returns each limit as
308
+ set, by label, so the worker can report them."""
309
+ limits = (
310
+ AddressSpaceLimit(resource.RLIMIT_AS, memory_mb << 20, "memory_bytes"),
311
+ ProcessLimit(resource.RLIMIT_FSIZE, file_mb << 20, "file_bytes"),
312
+ ProcessLimit(resource.RLIMIT_NOFILE, nofile, "open_files"),
313
+ ProcessLimit(resource.RLIMIT_CORE, 0, "core_bytes"),
314
+ )
315
+ return {limit.label: limit.apply() for limit in limits}
229
316
 
230
317
 
231
318
  def worker_environment(
@@ -295,18 +382,12 @@ class ConfinedVerifierSandbox(VerifierSandbox):
295
382
  return verifier_environment(home=home)
296
383
 
297
384
 
298
- # --------------------------------------------------------------------------- #
299
- # The supervisor's side: start, reach and stop workers.
300
- # --------------------------------------------------------------------------- #
301
-
302
-
303
385
  @dataclass
304
386
  class Worker:
305
387
  version_id: str
306
388
  process: subprocess.Popen
307
389
  socket_path: Path
308
390
  secret: str
309
- #: Sessions the supervisor has placed on this worker.
310
391
  sessions: set[str] = field(default_factory=set)
311
392
  #: When the worker last had a session, for idle reaping.
312
393
  idle_since: float = field(default_factory=time.time)
@@ -392,11 +473,15 @@ class WorkerPool:
392
473
  # left (sockets, session files) is unreachable.
393
474
  shutil.rmtree(self.workers_root, ignore_errors=True)
394
475
  self.workers_root.mkdir(parents=True, exist_ok=True, mode=0o700)
476
+ # AF_UNIX caps a socket path at 103 bytes on macOS (107 on Linux): the
477
+ # sockets live in a short private (0700) directory of their own, so no
478
+ # cache root or version id is too long to serve.
479
+ self._sockets = tempfile.TemporaryDirectory(prefix="gw-")
480
+ self.sockets = Path(self._sockets.name)
395
481
  self.worlds_domain = worlds_domain
396
482
  self.memory_mb = memory_mb
397
483
  self.budget_mb = budget_mb
398
484
  self.footprint_mb = footprint_mb
399
- #: Workers live at once: the budget's worth of footprints.
400
485
  self.max_workers = budget_mb // footprint_mb
401
486
  self.idle_s = idle_s
402
487
  self.python = python
@@ -475,7 +560,7 @@ class WorkerPool:
475
560
  def _spawn(self, version_id: str) -> Worker:
476
561
  home = self.workers_root / version_id
477
562
  home.mkdir(parents=True, exist_ok=True, mode=0o700)
478
- socket_path = home / "worker.sock"
563
+ socket_path = self.sockets / f"{hashlib.sha256(version_id.encode()).hexdigest()[:16]}.sock"
479
564
  secret = secrets.token_urlsafe(24)
480
565
  # The supervisor binds and listens; the worker inherits the descriptor
481
566
  # and serves on it, so the socket is the supervisor's from the start.