@withgateway/sdk 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/agents/index.d.mts +12 -4
  2. package/dist/agents/index.d.ts +12 -4
  3. package/dist/agents/index.js +30 -0
  4. package/dist/agents/index.js.map +1 -1
  5. package/dist/agents/index.mjs +30 -0
  6. package/dist/agents/index.mjs.map +1 -1
  7. package/dist/cli.js +2291 -945
  8. package/dist/cli.js.map +1 -1
  9. package/dist/cli.mjs +2290 -944
  10. package/dist/cli.mjs.map +1 -1
  11. package/dist/files/index.js.map +1 -1
  12. package/dist/files/index.mjs.map +1 -1
  13. package/dist/index.js +325 -184
  14. package/dist/index.js.map +1 -1
  15. package/dist/index.mjs +324 -183
  16. package/dist/index.mjs.map +1 -1
  17. package/dist/traces/index.js.map +1 -1
  18. package/dist/traces/index.mjs.map +1 -1
  19. package/dist/tracing/index.js +1 -1
  20. package/dist/tracing/index.js.map +1 -1
  21. package/dist/tracing/index.mjs +1 -1
  22. package/dist/tracing/index.mjs.map +1 -1
  23. package/dist/worlds/index.d.mts +3 -1
  24. package/dist/worlds/index.d.ts +3 -1
  25. package/dist/worlds/index.js +178 -67
  26. package/dist/worlds/index.js.map +1 -1
  27. package/dist/worlds/index.mjs +177 -66
  28. package/dist/worlds/index.mjs.map +1 -1
  29. package/package.json +1 -1
  30. package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
  31. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  32. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  33. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  34. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  35. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  36. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  37. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  38. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  39. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  40. package/runtime/gatewaysdk/environment/schema/host.py +21 -2
  41. package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
  42. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  43. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  44. package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
  45. package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
  46. package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
  47. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  48. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  49. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  51. package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
  52. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  53. package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
  54. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  55. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  56. package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
  57. package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
  58. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  59. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  60. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  61. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
  62. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  63. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  64. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
  65. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
  66. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  68. package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
  69. package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
  70. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  71. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  72. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  73. package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
  74. package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
  75. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  76. package/skills/connector-schema-authoring/SKILL.md +1 -0
  77. package/skills/connector-schema-authoring/connector-worlds.md +1 -0
  78. package/skills/connector-schema-authoring/world-schema.md +7 -0
  79. package/skills/factory-orchestrator/SKILL.md +147 -0
  80. package/skills/world-data-ingestion/ROWS.md +5 -1
  81. package/skills/world-data-ingestion/SKILL.md +4 -4
  82. package/skills/worlds-getting-started/SKILL.md +2 -2
  83. package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  84. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
@@ -0,0 +1,187 @@
1
+ """``MeasuredTestRun``: a world's tests run in a child interpreter the bridge measures from outside.
2
+
3
+ World code (its handlers, its tools, the engine under them) runs in that child, and the run's peak is
4
+ what the kernel reports for the child when the bridge reaps it (``wait4``'s ``ru_maxrss``: the larger
5
+ of the child and the pytest it ran). So code in the world can neither set nor lower the number, and
6
+ none of the bridge's own request decoding is in it; a ``memory`` the child's report carries is dropped.
7
+ A run that served nothing (no case ran, every Python test skipped) has no peak. The child leads its own
8
+ process group, takes no core dump (and on Linux is not dumpable), and has no stdio of the bridge's, so
9
+ the bridge's callers never wait on world code. When the bridge goes, the kernel kills the child (Linux)
10
+ and a sweeper the child forked before any world code ran, which shares no lock with it, kills the group
11
+ (its pytest and whatever the world started in it) once a lifeline descriptor the bridge holds closes. The
12
+ bridge kills the group itself once the child has exited (before reaping it where ``waitid`` exists).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import os
19
+ import resource
20
+ import signal
21
+ import stat
22
+ import subprocess
23
+ import sys
24
+ import tempfile
25
+ from contextlib import suppress
26
+ from dataclasses import dataclass
27
+ from pathlib import Path
28
+
29
+ import gatewaysdk
30
+
31
+ from .process_memory import maxrss_mb
32
+ from .validation import ContractError, fail
33
+ from .workers import ProcessLimit, die_with_parent, seal_process
34
+
35
+ _BOOT = (
36
+ "import runpy, sys, types; "
37
+ "package = types.ModuleType('gatewaysdk'); package.__path__ = [sys.argv[1]]; "
38
+ "sys.modules['gatewaysdk'] = package; "
39
+ "runpy.run_module('gatewaysdk.environment.schema.measured_tests', run_name='__main__')"
40
+ )
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class MeasuredTestRun:
45
+ report: dict
46
+ peak_mb: int | None
47
+
48
+ @classmethod
49
+ def of(cls, root: Path, scenario: str | None) -> MeasuredTestRun:
50
+ with tempfile.TemporaryDirectory(prefix="gateway-measured-") as directory:
51
+ answer = Path(directory) / "answer.json"
52
+ lifeline, held = os.pipe()
53
+ argv = [sys.executable, "-I", "-B", "-c", _BOOT, list(gatewaysdk.__path__)[0], str(os.getpid())]
54
+ try:
55
+ child = subprocess.Popen(
56
+ [*argv, str(lifeline), str(root), scenario or "", str(answer)],
57
+ stdin=subprocess.DEVNULL,
58
+ stdout=subprocess.DEVNULL,
59
+ stderr=subprocess.DEVNULL,
60
+ pass_fds=(lifeline,),
61
+ start_new_session=True,
62
+ )
63
+ finally:
64
+ os.close(lifeline)
65
+ try:
66
+ code, peak_mb = _reaped(child)
67
+ except BaseException:
68
+ _stop_group(child.pid)
69
+ raise
70
+ finally:
71
+ os.close(held)
72
+ return cls._read(answer, code, peak_mb)
73
+
74
+ @classmethod
75
+ def _read(cls, answer: Path, code: int, peak_mb: int | None) -> MeasuredTestRun:
76
+ result = _answer_of(answer)
77
+ if result is None:
78
+ reached = "" if peak_mb is None else f", having reached {peak_mb} MiB"
79
+ fail("tests", f"the world's test run stopped before it reported (exit {code}{reached})")
80
+ refusal = result.get("error")
81
+ if isinstance(refusal, dict):
82
+ raise _reported(refusal)
83
+ if not isinstance(result.get("report"), dict):
84
+ fail("tests", "the world's test run reported nothing readable")
85
+ report = {name: value for name, value in result["report"].items() if name != "memory"}
86
+ return cls(report, peak_mb if result.get("served") is True else None)
87
+
88
+
89
+ def _answer_of(answer: Path) -> dict | None:
90
+ """The child's answer: a regular file of at most the bridge's own answer limit, never followed through a
91
+ link or waited on as a pipe, since the child runs world code; None when there is none to read."""
92
+ from .platform import MAX_BYTES
93
+
94
+ flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
95
+ try:
96
+ fd = os.open(answer, flags)
97
+ except OSError:
98
+ return None
99
+ with os.fdopen(fd, "rb") as file:
100
+ if not stat.S_ISREG(os.fstat(file.fileno()).st_mode):
101
+ return None
102
+ raw = file.read(2 * MAX_BYTES + 1)
103
+ if len(raw) > 2 * MAX_BYTES:
104
+ fail("tests", "the world's test run reported more than the bridge answers")
105
+ try:
106
+ result = json.loads(raw)
107
+ except ValueError:
108
+ return None
109
+ return result if isinstance(result, dict) else None
110
+
111
+
112
+ def _reported(refusal: dict) -> ContractError:
113
+ """The refusal the child reported, rebuilt from its strings alone: the child runs world code."""
114
+ text = {name: str(refusal.get(name) or "") for name in ("code", "message", "schema_path", "path")}
115
+ return ContractError(text["code"], text["message"], schema_path=text["schema_path"], path=text["path"])
116
+
117
+
118
+ def _reaped(child: subprocess.Popen) -> tuple[int, int | None]:
119
+ """The child's exit code and, where ``wait4`` exists, its peak; whatever of its group still runs once it
120
+ has exited is killed."""
121
+ if not hasattr(os, "wait4"):
122
+ return child.wait(), None
123
+ stopped = _exited_unreaped(child.pid)
124
+ _, status, usage = os.wait4(child.pid, 0)
125
+ if not stopped:
126
+ _stop_group(child.pid)
127
+ child.returncode = os.waitstatus_to_exitcode(status)
128
+ return child.returncode, maxrss_mb(usage.ru_maxrss)
129
+
130
+
131
+ def _exited_unreaped(leader: int) -> bool:
132
+ """Waits for ``leader`` to exit without reaping it, then kills what of its group still runs, so the group
133
+ id cannot be another process's yet; False where there is no ``waitid`` (macOS), whose caller kills after."""
134
+ if not hasattr(os, "waitid"):
135
+ return False
136
+ os.waitid(os.P_PID, leader, os.WEXITED | os.WNOWAIT)
137
+ _stop_group(leader)
138
+ return True
139
+
140
+
141
+ def _stop_group(leader: int) -> None:
142
+ with suppress(ProcessLookupError, PermissionError, AttributeError):
143
+ os.killpg(leader, signal.SIGKILL)
144
+
145
+
146
+ def _sweeper(lifeline: int) -> int:
147
+ """Forks, while this process is still one thread with no world code loaded, a process of its group that
148
+ waits for the lifeline to close and then kills the group, itself included. A process of its own: the
149
+ kernel's kill of the child on the bridge's exit (which a fork does not inherit) would take a thread with
150
+ it, and no lock world code holds can delay it."""
151
+ sweeper = os.fork()
152
+ if sweeper == 0:
153
+ while os.read(lifeline, 1 << 16):
154
+ pass
155
+ os.killpg(0, signal.SIGKILL)
156
+ os._exit(1)
157
+ os.close(lifeline)
158
+ return sweeper
159
+
160
+
161
+ def _done_with(sweeper: int) -> None:
162
+ """The child reaps its own sweeper once it has answered, so no zombie waits on whatever PID 1 is."""
163
+ with suppress(ProcessLookupError, ChildProcessError):
164
+ os.kill(sweeper, signal.SIGKILL)
165
+ os.waitpid(sweeper, 0)
166
+
167
+
168
+ def _serve_and_report(bridge: int, lifeline: int, root: Path, scenario: str | None, answer: Path) -> None:
169
+ die_with_parent(bridge)
170
+ seal_process()
171
+ ProcessLimit(resource.RLIMIT_CORE, 0, "core_bytes").apply()
172
+ sweeper = _sweeper(lifeline)
173
+ from .world_tests import run
174
+
175
+ try:
176
+ report = run(root, scenario=scenario)
177
+ result = {"report": report.to_json(), "served": report.served}
178
+ except ContractError as error:
179
+ result = {"error": error.detail}
180
+ except Exception as error: # noqa: BLE001 - the child has no stderr; its answer is the only report it can give
181
+ result = {"error": {"code": "tests", "message": f"the world's test run failed: {type(error).__name__}"}}
182
+ answer.write_text(json.dumps(result), encoding="utf-8")
183
+ _done_with(sweeper)
184
+
185
+
186
+ if __name__ == "__main__":
187
+ _serve_and_report(int(sys.argv[2]), int(sys.argv[3]), Path(sys.argv[4]), sys.argv[5] or None, Path(sys.argv[6]))
@@ -67,6 +67,9 @@ COMMANDS = (
67
67
  "chunk-files",
68
68
  "ingest",
69
69
  "ingest-calls",
70
+ "map-rows",
71
+ "redact-rows",
72
+ "redact-policy",
70
73
  "conform",
71
74
  "reconcile",
72
75
  "counts",
@@ -560,6 +563,52 @@ def _ingest_calls(request: dict) -> dict:
560
563
  return answer
561
564
 
562
565
 
566
+ def _map_rows(request: dict) -> dict:
567
+ """Rows a declared ``ingest.toml`` maps the records to, for the Data Factory's upload, which lands them itself."""
568
+ from .ingest import map_records
569
+
570
+ records = request.get("records")
571
+ if not isinstance(records, list):
572
+ fail("input", "Expected records as a list of objects")
573
+ mapping = request.get("map")
574
+ if not isinstance(mapping, str):
575
+ fail("input", "map is an ingest.toml text")
576
+ with tempfile.TemporaryDirectory(prefix="gateway-map-rows-") as directory:
577
+ root = Path(directory) / "bundle"
578
+ root.mkdir()
579
+ _write_files(root, request.get("files"))
580
+ projected, report = map_records(world_root(root), records, mapping)
581
+ return {"status": "ok", "projected": projected, **report}
582
+
583
+
584
+ def _redact_rows(request: dict) -> dict:
585
+ """Rows under the world's ``[redact]`` policy, for the Data Factory's upload, which plans and checks them in the form the world stores."""
586
+ from .redaction import parse_mode, redact_rows, require_policy
587
+
588
+ rows = request.get("rows")
589
+ if not isinstance(rows, dict) or not all(isinstance(listed, list) for listed in rows.values()):
590
+ fail("input", "Expected rows as {entity: [rows]}")
591
+ mode = parse_mode(request.get("redact"))
592
+ with tempfile.TemporaryDirectory(prefix="gateway-redact-rows-") as directory:
593
+ root = Path(directory) / "bundle"
594
+ root.mkdir()
595
+ _write_files(root, request.get("files"))
596
+ redacted = redact_rows(require_policy(world_root(root), mode), rows, mode)
597
+ return {"status": "ok", "rows": redacted}
598
+
599
+
600
+ def _redact_policy(request: dict) -> dict:
601
+ """The ``[redact]`` policy this world's files declare, read exactly as ``redact-rows`` reads it (``policy_for``), or null when there is none."""
602
+ from .redaction import policy_for, policy_view
603
+
604
+ with tempfile.TemporaryDirectory(prefix="gateway-redact-policy-") as directory:
605
+ root = Path(directory) / "bundle"
606
+ root.mkdir()
607
+ _write_files(root, request.get("files"))
608
+ policy = policy_view(policy_for(world_root(root)))
609
+ return {"status": "ok", "policy": policy}
610
+
611
+
563
612
  def _workdir(request: dict) -> Path:
564
613
  workdir = request.get("workdir")
565
614
  if not isinstance(workdir, str) or not workdir or not Path(workdir).is_absolute() or not Path(workdir).is_dir():
@@ -821,8 +870,8 @@ def _conform(request: dict) -> dict:
821
870
 
822
871
 
823
872
  def _run_tests(request: dict) -> dict:
873
+ from .measured_tests import MeasuredTestRun
824
874
  from .snapshot import fetch as fetch_snapshot
825
- from .world_tests import run
826
875
 
827
876
  scenario = request.get("scenario")
828
877
  if scenario is not None and not isinstance(scenario, str):
@@ -836,8 +885,10 @@ def _run_tests(request: dict) -> dict:
836
885
  root = _world_for(request, Path(directory))
837
886
  if snapshot is not None:
838
887
  fetch_snapshot(root, check_bundle(root), snapshot["url"], snapshot["sha256"], snapshot["digest"])
839
- report = run(root, scenario=scenario).to_json()
840
- report["world"] = root.name
888
+ measured = MeasuredTestRun.of(root, scenario)
889
+ report = {**measured.report, "world": root.name}
890
+ if measured.peak_mb is not None:
891
+ report["memory"] = {"peakMb": measured.peak_mb}
841
892
  return {"status": "ok", "report": report}
842
893
 
843
894
 
@@ -978,6 +1029,9 @@ def _answering_commands() -> dict:
978
1029
  "chunk-files": lambda request: platform_import.chunk_files(request, _world_for),
979
1030
  "ingest": _ingest,
980
1031
  "ingest-calls": _ingest_calls,
1032
+ "map-rows": _map_rows,
1033
+ "redact-rows": _redact_rows,
1034
+ "redact-policy": _redact_policy,
981
1035
  "conform": _conform,
982
1036
  "reconcile": _reconcile,
983
1037
  "run-tests": _run_tests,
@@ -20,6 +20,7 @@ from typing import Any, Callable
20
20
  from ..legacy_sqlite import LegacyVerifiers
21
21
  from .compiler import bundle_mode
22
22
  from .data import QUERY_DEFAULT_LIMIT, entity_counts, entity_query, entity_query_across, entity_rows, world_kind
23
+ from .process_memory import SIZE, ProcessStatus
23
24
  from .validation import ContractError, fail
24
25
  from .world_sql import WorldSql
25
26
 
@@ -127,10 +128,10 @@ def cap_memory_for(request: Any, status: Path = Path("/proc/self/status"), limit
127
128
  return
128
129
  if limits is None:
129
130
  import resource as limits
130
- sizes = [int(line.split()[1]) * 1024 for line in status.read_text().splitlines() if line.startswith("VmSize:")]
131
- if not sizes:
131
+ size = ProcessStatus.read(status).bytes(SIZE)
132
+ if size is None:
132
133
  fail("runtime", "This process cannot read its own memory size, so it refuses to run caller SQL")
133
- cap = sizes[0] + SQL_MEMORY_BYTES
134
+ cap = size + SQL_MEMORY_BYTES
134
135
  soft, hard = limits.getrlimit(limits.RLIMIT_AS)
135
136
  hard = _lower(cap, hard, limits.RLIM_INFINITY)
136
137
  limits.setrlimit(limits.RLIMIT_AS, (_lower(hard, soft, limits.RLIM_INFINITY), hard))
@@ -10,15 +10,19 @@ from __future__ import annotations
10
10
 
11
11
  import json
12
12
  import tempfile
13
- from pathlib import Path
13
+ from dataclasses import replace
14
+ from pathlib import Path, PurePosixPath
14
15
  from typing import TYPE_CHECKING, Any, Callable
15
16
 
16
17
  from .cast import world_fields
18
+ from .shape_catalog import CatalogWorld, ShapeCatalog
19
+ from .shape_declarations import answer_declarations
17
20
  from .scaffold import (
18
21
  CUSTOM,
19
22
  TEMPLATE_SLUG,
20
23
  CustomScaffold,
21
24
  TemplateWorld,
25
+ WorldTemplate,
22
26
  init_bundle,
23
27
  init_bundle_from_files,
24
28
  init_bundle_from_world,
@@ -26,7 +30,7 @@ from .scaffold import (
26
30
  request_files,
27
31
  template_digest,
28
32
  )
29
- from .validation import ContractError, fail
33
+ from .validation import ContractError, fail, parse_toml
30
34
 
31
35
  if TYPE_CHECKING:
32
36
  from .connector import Spec
@@ -164,6 +168,12 @@ def _operations(spec: "Spec") -> list[dict]:
164
168
  ]
165
169
 
166
170
 
171
+ def _redacts(policy: "Spec | RedactPolicy | None") -> bool:
172
+ if policy is None:
173
+ return False
174
+ return bool(policy.hash_fields or policy.drop_fields or policy.preserve_fields or policy.round_fields)
175
+
176
+
167
177
  def _served_operations(spec: "Spec", settings: Any) -> list[dict]:
168
178
  operations = _operations(spec)
169
179
  for operation in operations:
@@ -173,13 +183,13 @@ def _served_operations(spec: "Spec", settings: Any) -> list[dict]:
173
183
  return operations
174
184
 
175
185
 
176
- def template_summary(files: dict[str, bytes]) -> dict:
186
+ def template_summary(files: dict[str, bytes], resolved: CatalogWorld | ContractError | None = None) -> dict:
177
187
  """Never runs the template's Python."""
178
188
  from .connector import bind_spec, parse_spec
179
189
  from .redaction import parse_env_policy
180
190
 
181
- world_path = next((path for path in ("world.json", "schema/world.json") if path in files), None)
182
- if world_path is None:
191
+ world_path = "schema/world.json" if _world_tree(files) else "world.json"
192
+ if world_path not in files:
183
193
  fail(
184
194
  "template",
185
195
  "No world.json (template) or schema/world.json (world) among the files",
@@ -196,7 +206,8 @@ def template_summary(files: dict[str, bytes]) -> dict:
196
206
  if policy is None and "gateway-env.toml" in files:
197
207
  policy = parse_env_policy(files["gateway-env.toml"].decode("utf-8"))
198
208
  entities = []
199
- fields_by_entity = world_fields(world)
209
+ resolved = resolved if resolved is not None else _resolved_files(files)
210
+ fields_by_entity = resolved.entity_fields() if isinstance(resolved, CatalogWorld) else world_fields(world)
200
211
  for name, entity in sorted((gateway.get("entities") or {}).items()):
201
212
  entities.append(
202
213
  {
@@ -235,23 +246,26 @@ def template_summary(files: dict[str, bytes]) -> dict:
235
246
 
236
247
 
237
248
  def describe(request: dict) -> dict:
238
- """``connector`` (``custom`` only) or the ``files`` of a template or a compiled world."""
249
+ """``connector`` (``custom`` only) or the ``files`` of a template or a compiled world;
250
+ ``entity`` narrows the description to that entity's fields and catalog entries."""
239
251
  from .platform import describe_dependencies
252
+ from .shape_catalog import narrowed_description
240
253
  from .ui import ui_kind
241
254
 
242
255
  with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
243
256
  root = Path(directory) / "bundle"
244
257
  authored = _described_tree(request, root)
245
- summary = template_summary({"world.json": (root / "schema/world.json").read_bytes()})["summary"]
258
+ resolved = _resolved_world(root)
259
+ summary = template_summary({"world.json": (root / "schema/world.json").read_bytes()}, resolved)["summary"]
246
260
  summary["digest"] = template_digest(authored)
247
261
  summary["files"] = sorted(authored)
248
262
  summary["ui"] = ui_kind(root)
249
263
  summary["tools"] = _tool_schemas(root)
250
264
  summary["connection"] = None
251
265
  summary["openapi"] = None
252
- _describe_surface(root, summary)
266
+ _describe_surface(root, summary, resolved)
253
267
  summary["dependencies"] = describe_dependencies(root) if (root / "gateway-env.toml").is_file() else []
254
- return {"status": "ok", "description": summary}
268
+ return {"status": "ok", "description": narrowed_description(summary, request.get("entity"))}
255
269
 
256
270
 
257
271
  def _described_tree(request: dict, root: Path) -> dict[str, bytes]:
@@ -261,7 +275,7 @@ def _described_tree(request: dict, root: Path) -> dict[str, bytes]:
261
275
  if connector is not None:
262
276
  return _custom_tree(connector, root)
263
277
  authored = request_files(request)
264
- if all(marker in authored for marker in COMPILED_MARKERS):
278
+ if _world_tree(authored):
265
279
  _write_compiled(authored, root)
266
280
  return authored
267
281
  slug = request.get("slug") if isinstance(request.get("slug"), str) else None
@@ -282,13 +296,31 @@ def _custom_tree(connector: Any, root: Path) -> dict[str, bytes]:
282
296
 
283
297
  def _write_compiled(authored: dict[str, bytes], root: Path) -> None:
284
298
  root.mkdir()
299
+ laid_out = set(DESCRIBE_PATHS) | _settings_paths(authored.get("connector.toml"))
285
300
  for path, data in authored.items():
286
- if path in DESCRIBE_PATHS:
301
+ if path in laid_out or _schema_sibling(path):
287
302
  target = root / path
288
303
  target.parent.mkdir(parents=True, exist_ok=True)
289
304
  target.write_bytes(data)
290
305
 
291
306
 
307
+ def _schema_sibling(path: str) -> bool:
308
+ return path.startswith("schema/") and path.count("/") == 1 and path.endswith(".json")
309
+
310
+
311
+ def _settings_paths(spec: bytes | None) -> set[str]:
312
+ """What ``load_api_settings`` looks for beside connector.toml: the handlers module it names and
313
+ the built ui's index, so a compiled world is described with the settings it is served with."""
314
+ from .api import FILTER_KEY, HANDLERS
315
+
316
+ raw = parse_toml(spec.decode("utf-8")) if spec is not None else {}
317
+ api = raw.get(FILTER_KEY) if isinstance(raw.get(FILTER_KEY), dict) else {}
318
+ ui = raw.get("ui") if isinstance(raw.get("ui"), dict) else {}
319
+ handlers = PurePosixPath(api.get("handlers") if isinstance(api.get("handlers"), str) else HANDLERS).as_posix()
320
+ static = ui.get("static")
321
+ return {handlers, (PurePosixPath(static) / "index.html").as_posix()} if isinstance(static, str) else {handlers}
322
+
323
+
292
324
  def _tool_schemas(root: Path) -> list[dict]:
293
325
  tool_schemas = {}
294
326
  if (root / "tool-schemas.json").is_file():
@@ -304,7 +336,7 @@ def _tool_schemas(root: Path) -> list[dict]:
304
336
  ]
305
337
 
306
338
 
307
- def _describe_surface(root: Path, summary: dict) -> None:
339
+ def _describe_surface(root: Path, summary: dict, resolved: CatalogWorld | ContractError) -> None:
308
340
  from .api import load_api_settings, openapi_document
309
341
  from .connector import load_spec
310
342
  from .redaction import env_policy
@@ -313,14 +345,15 @@ def _describe_surface(root: Path, summary: dict) -> None:
313
345
  policy = env_policy(root / "gateway-env.toml")
314
346
  _annotate_redaction(summary["entities"], policy)
315
347
  summary["redact"] = _redact_summary(policy) if policy else None
348
+ _catalog(summary, lambda: replace(_unrefused(resolved), redacted=_redacts(policy)))
316
349
  return
317
350
  spec = load_spec(root)
318
351
  try:
319
- settings = load_api_settings(root, spec)
320
- except ContractError:
321
- # A template without its handlers still has a shape worth showing.
322
- settings = None
352
+ settings, refused = load_api_settings(root, spec), None
353
+ except ContractError as error:
354
+ settings, refused = None, error
323
355
  _annotate_redaction(summary["entities"], spec)
356
+ _catalog(summary, lambda: _served_world(root, spec, settings, refused, resolved))
324
357
  summary["connection"] = {
325
358
  "slug": spec.slug,
326
359
  "baseUrl": spec.base_url,
@@ -330,3 +363,58 @@ def _describe_surface(root: Path, summary: dict) -> None:
330
363
  }
331
364
  summary["openapi"] = openapi_document(spec, summary.get("domainId") or spec.slug, settings)
332
365
  summary["redact"] = summary["connection"]["redact"]
366
+
367
+
368
+ def _served_world(
369
+ root: Path, spec: "Spec", settings: Any, refused: ContractError | None, resolved: CatalogWorld | ContractError
370
+ ) -> CatalogWorld:
371
+ if refused is not None:
372
+ raise refused
373
+ answers = answer_declarations(parse_toml((root / "connector.toml").read_text(encoding="utf-8")))
374
+ return replace(_unrefused(resolved), spec=spec, settings=settings, answers=answers, redacted=_redacts(spec))
375
+
376
+
377
+ def _resolved_world(root: Path) -> CatalogWorld | ContractError:
378
+ try:
379
+ return CatalogWorld.of_bundle(root)
380
+ except ContractError as refused:
381
+ return refused
382
+
383
+
384
+ def _unrefused(resolved: CatalogWorld | ContractError) -> CatalogWorld:
385
+ if isinstance(resolved, ContractError):
386
+ raise resolved
387
+ return resolved
388
+
389
+
390
+ def _resolved_files(files: dict[str, bytes]) -> CatalogWorld | ContractError:
391
+ with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
392
+ root = Path(directory) / "bundle"
393
+ try:
394
+ _lay_out_schema(files, root)
395
+ except ContractError as refused:
396
+ return refused
397
+ return _resolved_world(root)
398
+
399
+
400
+ def _lay_out_schema(files: dict[str, bytes], root: Path) -> None:
401
+ if _world_tree(files):
402
+ _write_compiled(files, root)
403
+ return
404
+ WorldTemplate(files).lay_out_schema(root)
405
+
406
+
407
+ def _world_tree(files: dict[str, bytes]) -> bool:
408
+ """A built world's files, whole or only their contract half (no ``tool-schemas.json``); else a template's."""
409
+ compiled = all(marker in files for marker in COMPILED_MARKERS)
410
+ return compiled or ("schema/world.json" in files and "world.json" not in files)
411
+
412
+
413
+ def _catalog(summary: dict, world: Callable[[], CatalogWorld]) -> None:
414
+ """The catalog describes the world as it is served. When that cannot be read the description
415
+ keeps its entities and says why under ``catalogRefused``, rather than answer an empty catalog."""
416
+ try:
417
+ summary["catalog"] = ShapeCatalog.of(world()).document()
418
+ except ContractError as error:
419
+ summary["catalog"] = None
420
+ summary["catalogRefused"] = error.detail
@@ -0,0 +1,76 @@
1
+ """A process's memory as its kernel counts it: the ``Vm*`` and ``Rss*`` lines of ``/proc/<pid>/status``,
2
+ and a reaped child's ``ru_maxrss``.
3
+
4
+ Read from the kernel, never asked of the process, so a worker running tenant code cannot
5
+ under-report what it holds. Where there is no such file (no ``/proc``, as on macOS, or a
6
+ process already gone) every figure is ``None`` and the caller falls back to what it was told.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+ import sys
13
+ from dataclasses import dataclass, field
14
+ from pathlib import Path
15
+
16
+ SIZE = "VmSize"
17
+ PEAK_SIZE = "VmPeak"
18
+ RESIDENT = "VmRSS"
19
+ RESIDENT_ANON = "RssAnon"
20
+ PEAK_RESIDENT = "VmHWM"
21
+ #: What a world's measured peak is multiplied by for what it needs of a worker (``conformance/world-sessions.json``).
22
+ HEADROOM = 1.5
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class ProcessStatus:
27
+ fields: dict[str, int] = field(default_factory=dict)
28
+
29
+ @classmethod
30
+ def read(cls, path: Path) -> ProcessStatus:
31
+ try:
32
+ text = path.read_text()
33
+ except OSError:
34
+ return cls()
35
+ return cls(dict(reading for line in text.splitlines() if (reading := _kib_line(line)) is not None))
36
+
37
+ def bytes(self, name: str) -> int | None:
38
+ return self.fields.get(name)
39
+
40
+ def mb(self, name: str) -> int | None:
41
+ value = self.fields.get(name)
42
+ return None if value is None else value >> 20
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class MemoryNeed:
47
+ """What a world needs of a worker: its measured peak and half again."""
48
+
49
+ peak_mb: int
50
+
51
+ @property
52
+ def mb(self) -> int:
53
+ return math.ceil(self.peak_mb * HEADROOM)
54
+
55
+
56
+ class ProcessMemory:
57
+ """Where ``/proc`` is read; a test hands it a directory laid out the same way."""
58
+
59
+ def __init__(self, proc: Path = Path("/proc")) -> None:
60
+ self.proc = proc
61
+
62
+ def status(self, pid: int | str = "self") -> ProcessStatus:
63
+ return ProcessStatus.read(self.proc / str(pid) / "status")
64
+
65
+
66
+ def maxrss_mb(maxrss: int) -> int:
67
+ """A ``ru_maxrss`` the kernel reported (KiB on Linux, bytes on macOS) in MiB."""
68
+ return maxrss >> 20 if sys.platform == "darwin" else maxrss >> 10
69
+
70
+
71
+ def _kib_line(line: str) -> tuple[str, int] | None:
72
+ name, _, rest = line.partition(":")
73
+ parts = rest.split()
74
+ if not name.startswith(("Vm", "Rss")) or len(parts) != 2 or parts[1] != "kB" or not parts[0].isdigit():
75
+ return None
76
+ return name, int(parts[0]) << 10
@@ -82,6 +82,19 @@ def policy_for(root: str | Path) -> Any | None:
82
82
  return env_policy(Path(root) / ENV_FILE)
83
83
 
84
84
 
85
+ def policy_view(spec: Any | None) -> dict | None:
86
+ """What a policy does to rows, as plain JSON, so two versions of a world can be compared: the salt and each field list."""
87
+ if spec is None:
88
+ return None
89
+ return {
90
+ "salt": spec.slug,
91
+ "hash": sorted(spec.hash_fields),
92
+ "drop": sorted(spec.drop_fields),
93
+ "round": dict(sorted(spec.round_fields.items())),
94
+ "preserve": sorted(spec.preserve_fields),
95
+ }
96
+
97
+
85
98
  def env_policy(path: Path) -> RedactPolicy | None:
86
99
  """``[redact]`` out of a ``gateway-env.toml``, or None when the file or the table is absent."""
87
100
  if not path.is_file():
@@ -77,6 +77,11 @@ class WorldRuntime(ABC):
77
77
  def authored_code(cls, cached: CachedVersion) -> str | None:
78
78
  """World-authored code ``cached`` runs, so only a version worker may open it; None when it runs none."""
79
79
 
80
+ @classmethod
81
+ @abstractmethod
82
+ def world_name(cls, cached: CachedVersion) -> str:
83
+ """The name ``cached`` gives its world, for what the host logs about it."""
84
+
80
85
  @classmethod
81
86
  def declared_dependencies(cls, cached: CachedVersion) -> dict[str, Dependency]:
82
87
  return {}
@@ -375,6 +380,11 @@ class TreeRuntime(WorldRuntime):
375
380
  def authored_code(cls, cached: CachedVersion) -> str:
376
381
  return "schema world handlers"
377
382
 
383
+ @classmethod
384
+ def world_name(cls, cached: CachedVersion) -> str:
385
+ assert cached.tools is not None
386
+ return cached.tools.manifest.name
387
+
378
388
  @classmethod
379
389
  def declared_dependencies(cls, cached: CachedVersion) -> dict[str, Dependency]:
380
390
  assert cached.tools is not None
@@ -522,6 +532,10 @@ class SnapshotRuntime(WorldRuntime):
522
532
  return None
523
533
  return f"api handlers module {cached.settings.handlers.name}"
524
534
 
535
+ @classmethod
536
+ def world_name(cls, cached: CachedVersion) -> str:
537
+ return cached.spec.slug if cached.spec is not None else cached.version_id
538
+
525
539
  @staticmethod
526
540
  def live_state(session_dir: Path) -> Path:
527
541
  return HostWorldState.live_state(session_dir)