@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""``MeasuredTestRun``: a world's tests run in a child interpreter the bridge measures from outside.
|
|
2
|
+
|
|
3
|
+
World code (its handlers, its tools, the engine under them) runs in that child, and the run's peak is
|
|
4
|
+
what the kernel reports for the child when the bridge reaps it (``wait4``'s ``ru_maxrss``: the larger
|
|
5
|
+
of the child and the pytest it ran). So code in the world can neither set nor lower the number, and
|
|
6
|
+
none of the bridge's own request decoding is in it; a ``memory`` the child's report carries is dropped.
|
|
7
|
+
A run that served nothing (no case ran, every Python test skipped) has no peak. The child leads its own
|
|
8
|
+
process group, takes no core dump (and on Linux is not dumpable), and has no stdio of the bridge's, so
|
|
9
|
+
the bridge's callers never wait on world code. When the bridge goes, the kernel kills the child (Linux)
|
|
10
|
+
and a sweeper the child forked before any world code ran, which shares no lock with it, kills the group
|
|
11
|
+
(its pytest and whatever the world started in it) once a lifeline descriptor the bridge holds closes. The
|
|
12
|
+
bridge kills the group itself once the child has exited (before reaping it where ``waitid`` exists).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import resource
|
|
20
|
+
import signal
|
|
21
|
+
import stat
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import tempfile
|
|
25
|
+
from contextlib import suppress
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
import gatewaysdk
|
|
30
|
+
|
|
31
|
+
from .process_memory import maxrss_mb
|
|
32
|
+
from .validation import ContractError, fail
|
|
33
|
+
from .workers import ProcessLimit, die_with_parent, seal_process
|
|
34
|
+
|
|
35
|
+
_BOOT = (
|
|
36
|
+
"import runpy, sys, types; "
|
|
37
|
+
"package = types.ModuleType('gatewaysdk'); package.__path__ = [sys.argv[1]]; "
|
|
38
|
+
"sys.modules['gatewaysdk'] = package; "
|
|
39
|
+
"runpy.run_module('gatewaysdk.environment.schema.measured_tests', run_name='__main__')"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class MeasuredTestRun:
|
|
45
|
+
report: dict
|
|
46
|
+
peak_mb: int | None
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def of(cls, root: Path, scenario: str | None) -> MeasuredTestRun:
|
|
50
|
+
with tempfile.TemporaryDirectory(prefix="gateway-measured-") as directory:
|
|
51
|
+
answer = Path(directory) / "answer.json"
|
|
52
|
+
lifeline, held = os.pipe()
|
|
53
|
+
argv = [sys.executable, "-I", "-B", "-c", _BOOT, list(gatewaysdk.__path__)[0], str(os.getpid())]
|
|
54
|
+
try:
|
|
55
|
+
child = subprocess.Popen(
|
|
56
|
+
[*argv, str(lifeline), str(root), scenario or "", str(answer)],
|
|
57
|
+
stdin=subprocess.DEVNULL,
|
|
58
|
+
stdout=subprocess.DEVNULL,
|
|
59
|
+
stderr=subprocess.DEVNULL,
|
|
60
|
+
pass_fds=(lifeline,),
|
|
61
|
+
start_new_session=True,
|
|
62
|
+
)
|
|
63
|
+
finally:
|
|
64
|
+
os.close(lifeline)
|
|
65
|
+
try:
|
|
66
|
+
code, peak_mb = _reaped(child)
|
|
67
|
+
except BaseException:
|
|
68
|
+
_stop_group(child.pid)
|
|
69
|
+
raise
|
|
70
|
+
finally:
|
|
71
|
+
os.close(held)
|
|
72
|
+
return cls._read(answer, code, peak_mb)
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def _read(cls, answer: Path, code: int, peak_mb: int | None) -> MeasuredTestRun:
|
|
76
|
+
result = _answer_of(answer)
|
|
77
|
+
if result is None:
|
|
78
|
+
reached = "" if peak_mb is None else f", having reached {peak_mb} MiB"
|
|
79
|
+
fail("tests", f"the world's test run stopped before it reported (exit {code}{reached})")
|
|
80
|
+
refusal = result.get("error")
|
|
81
|
+
if isinstance(refusal, dict):
|
|
82
|
+
raise _reported(refusal)
|
|
83
|
+
if not isinstance(result.get("report"), dict):
|
|
84
|
+
fail("tests", "the world's test run reported nothing readable")
|
|
85
|
+
report = {name: value for name, value in result["report"].items() if name != "memory"}
|
|
86
|
+
return cls(report, peak_mb if result.get("served") is True else None)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _answer_of(answer: Path) -> dict | None:
|
|
90
|
+
"""The child's answer: a regular file of at most the bridge's own answer limit, never followed through a
|
|
91
|
+
link or waited on as a pipe, since the child runs world code; None when there is none to read."""
|
|
92
|
+
from .platform import MAX_BYTES
|
|
93
|
+
|
|
94
|
+
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
|
|
95
|
+
try:
|
|
96
|
+
fd = os.open(answer, flags)
|
|
97
|
+
except OSError:
|
|
98
|
+
return None
|
|
99
|
+
with os.fdopen(fd, "rb") as file:
|
|
100
|
+
if not stat.S_ISREG(os.fstat(file.fileno()).st_mode):
|
|
101
|
+
return None
|
|
102
|
+
raw = file.read(2 * MAX_BYTES + 1)
|
|
103
|
+
if len(raw) > 2 * MAX_BYTES:
|
|
104
|
+
fail("tests", "the world's test run reported more than the bridge answers")
|
|
105
|
+
try:
|
|
106
|
+
result = json.loads(raw)
|
|
107
|
+
except ValueError:
|
|
108
|
+
return None
|
|
109
|
+
return result if isinstance(result, dict) else None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _reported(refusal: dict) -> ContractError:
|
|
113
|
+
"""The refusal the child reported, rebuilt from its strings alone: the child runs world code."""
|
|
114
|
+
text = {name: str(refusal.get(name) or "") for name in ("code", "message", "schema_path", "path")}
|
|
115
|
+
return ContractError(text["code"], text["message"], schema_path=text["schema_path"], path=text["path"])
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _reaped(child: subprocess.Popen) -> tuple[int, int | None]:
|
|
119
|
+
"""The child's exit code and, where ``wait4`` exists, its peak; whatever of its group still runs once it
|
|
120
|
+
has exited is killed."""
|
|
121
|
+
if not hasattr(os, "wait4"):
|
|
122
|
+
return child.wait(), None
|
|
123
|
+
stopped = _exited_unreaped(child.pid)
|
|
124
|
+
_, status, usage = os.wait4(child.pid, 0)
|
|
125
|
+
if not stopped:
|
|
126
|
+
_stop_group(child.pid)
|
|
127
|
+
child.returncode = os.waitstatus_to_exitcode(status)
|
|
128
|
+
return child.returncode, maxrss_mb(usage.ru_maxrss)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _exited_unreaped(leader: int) -> bool:
|
|
132
|
+
"""Waits for ``leader`` to exit without reaping it, then kills what of its group still runs, so the group
|
|
133
|
+
id cannot be another process's yet; False where there is no ``waitid`` (macOS), whose caller kills after."""
|
|
134
|
+
if not hasattr(os, "waitid"):
|
|
135
|
+
return False
|
|
136
|
+
os.waitid(os.P_PID, leader, os.WEXITED | os.WNOWAIT)
|
|
137
|
+
_stop_group(leader)
|
|
138
|
+
return True
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _stop_group(leader: int) -> None:
|
|
142
|
+
with suppress(ProcessLookupError, PermissionError, AttributeError):
|
|
143
|
+
os.killpg(leader, signal.SIGKILL)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _sweeper(lifeline: int) -> int:
|
|
147
|
+
"""Forks, while this process is still one thread with no world code loaded, a process of its group that
|
|
148
|
+
waits for the lifeline to close and then kills the group, itself included. A process of its own: the
|
|
149
|
+
kernel's kill of the child on the bridge's exit (which a fork does not inherit) would take a thread with
|
|
150
|
+
it, and no lock world code holds can delay it."""
|
|
151
|
+
sweeper = os.fork()
|
|
152
|
+
if sweeper == 0:
|
|
153
|
+
while os.read(lifeline, 1 << 16):
|
|
154
|
+
pass
|
|
155
|
+
os.killpg(0, signal.SIGKILL)
|
|
156
|
+
os._exit(1)
|
|
157
|
+
os.close(lifeline)
|
|
158
|
+
return sweeper
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _done_with(sweeper: int) -> None:
|
|
162
|
+
"""The child reaps its own sweeper once it has answered, so no zombie waits on whatever PID 1 is."""
|
|
163
|
+
with suppress(ProcessLookupError, ChildProcessError):
|
|
164
|
+
os.kill(sweeper, signal.SIGKILL)
|
|
165
|
+
os.waitpid(sweeper, 0)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _serve_and_report(bridge: int, lifeline: int, root: Path, scenario: str | None, answer: Path) -> None:
|
|
169
|
+
die_with_parent(bridge)
|
|
170
|
+
seal_process()
|
|
171
|
+
ProcessLimit(resource.RLIMIT_CORE, 0, "core_bytes").apply()
|
|
172
|
+
sweeper = _sweeper(lifeline)
|
|
173
|
+
from .world_tests import run
|
|
174
|
+
|
|
175
|
+
try:
|
|
176
|
+
report = run(root, scenario=scenario)
|
|
177
|
+
result = {"report": report.to_json(), "served": report.served}
|
|
178
|
+
except ContractError as error:
|
|
179
|
+
result = {"error": error.detail}
|
|
180
|
+
except Exception as error: # noqa: BLE001 - the child has no stderr; its answer is the only report it can give
|
|
181
|
+
result = {"error": {"code": "tests", "message": f"the world's test run failed: {type(error).__name__}"}}
|
|
182
|
+
answer.write_text(json.dumps(result), encoding="utf-8")
|
|
183
|
+
_done_with(sweeper)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
if __name__ == "__main__":
|
|
187
|
+
_serve_and_report(int(sys.argv[2]), int(sys.argv[3]), Path(sys.argv[4]), sys.argv[5] or None, Path(sys.argv[6]))
|
|
@@ -67,6 +67,9 @@ COMMANDS = (
|
|
|
67
67
|
"chunk-files",
|
|
68
68
|
"ingest",
|
|
69
69
|
"ingest-calls",
|
|
70
|
+
"map-rows",
|
|
71
|
+
"redact-rows",
|
|
72
|
+
"redact-policy",
|
|
70
73
|
"conform",
|
|
71
74
|
"reconcile",
|
|
72
75
|
"counts",
|
|
@@ -560,6 +563,52 @@ def _ingest_calls(request: dict) -> dict:
|
|
|
560
563
|
return answer
|
|
561
564
|
|
|
562
565
|
|
|
566
|
+
def _map_rows(request: dict) -> dict:
|
|
567
|
+
"""Rows a declared ``ingest.toml`` maps the records to, for the Data Factory's upload, which lands them itself."""
|
|
568
|
+
from .ingest import map_records
|
|
569
|
+
|
|
570
|
+
records = request.get("records")
|
|
571
|
+
if not isinstance(records, list):
|
|
572
|
+
fail("input", "Expected records as a list of objects")
|
|
573
|
+
mapping = request.get("map")
|
|
574
|
+
if not isinstance(mapping, str):
|
|
575
|
+
fail("input", "map is an ingest.toml text")
|
|
576
|
+
with tempfile.TemporaryDirectory(prefix="gateway-map-rows-") as directory:
|
|
577
|
+
root = Path(directory) / "bundle"
|
|
578
|
+
root.mkdir()
|
|
579
|
+
_write_files(root, request.get("files"))
|
|
580
|
+
projected, report = map_records(world_root(root), records, mapping)
|
|
581
|
+
return {"status": "ok", "projected": projected, **report}
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _redact_rows(request: dict) -> dict:
|
|
585
|
+
"""Rows under the world's ``[redact]`` policy, for the Data Factory's upload, which plans and checks them in the form the world stores."""
|
|
586
|
+
from .redaction import parse_mode, redact_rows, require_policy
|
|
587
|
+
|
|
588
|
+
rows = request.get("rows")
|
|
589
|
+
if not isinstance(rows, dict) or not all(isinstance(listed, list) for listed in rows.values()):
|
|
590
|
+
fail("input", "Expected rows as {entity: [rows]}")
|
|
591
|
+
mode = parse_mode(request.get("redact"))
|
|
592
|
+
with tempfile.TemporaryDirectory(prefix="gateway-redact-rows-") as directory:
|
|
593
|
+
root = Path(directory) / "bundle"
|
|
594
|
+
root.mkdir()
|
|
595
|
+
_write_files(root, request.get("files"))
|
|
596
|
+
redacted = redact_rows(require_policy(world_root(root), mode), rows, mode)
|
|
597
|
+
return {"status": "ok", "rows": redacted}
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _redact_policy(request: dict) -> dict:
|
|
601
|
+
"""The ``[redact]`` policy this world's files declare, read exactly as ``redact-rows`` reads it (``policy_for``), or null when there is none."""
|
|
602
|
+
from .redaction import policy_for, policy_view
|
|
603
|
+
|
|
604
|
+
with tempfile.TemporaryDirectory(prefix="gateway-redact-policy-") as directory:
|
|
605
|
+
root = Path(directory) / "bundle"
|
|
606
|
+
root.mkdir()
|
|
607
|
+
_write_files(root, request.get("files"))
|
|
608
|
+
policy = policy_view(policy_for(world_root(root)))
|
|
609
|
+
return {"status": "ok", "policy": policy}
|
|
610
|
+
|
|
611
|
+
|
|
563
612
|
def _workdir(request: dict) -> Path:
|
|
564
613
|
workdir = request.get("workdir")
|
|
565
614
|
if not isinstance(workdir, str) or not workdir or not Path(workdir).is_absolute() or not Path(workdir).is_dir():
|
|
@@ -821,8 +870,8 @@ def _conform(request: dict) -> dict:
|
|
|
821
870
|
|
|
822
871
|
|
|
823
872
|
def _run_tests(request: dict) -> dict:
|
|
873
|
+
from .measured_tests import MeasuredTestRun
|
|
824
874
|
from .snapshot import fetch as fetch_snapshot
|
|
825
|
-
from .world_tests import run
|
|
826
875
|
|
|
827
876
|
scenario = request.get("scenario")
|
|
828
877
|
if scenario is not None and not isinstance(scenario, str):
|
|
@@ -836,8 +885,10 @@ def _run_tests(request: dict) -> dict:
|
|
|
836
885
|
root = _world_for(request, Path(directory))
|
|
837
886
|
if snapshot is not None:
|
|
838
887
|
fetch_snapshot(root, check_bundle(root), snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
839
|
-
|
|
840
|
-
report
|
|
888
|
+
measured = MeasuredTestRun.of(root, scenario)
|
|
889
|
+
report = {**measured.report, "world": root.name}
|
|
890
|
+
if measured.peak_mb is not None:
|
|
891
|
+
report["memory"] = {"peakMb": measured.peak_mb}
|
|
841
892
|
return {"status": "ok", "report": report}
|
|
842
893
|
|
|
843
894
|
|
|
@@ -978,6 +1029,9 @@ def _answering_commands() -> dict:
|
|
|
978
1029
|
"chunk-files": lambda request: platform_import.chunk_files(request, _world_for),
|
|
979
1030
|
"ingest": _ingest,
|
|
980
1031
|
"ingest-calls": _ingest_calls,
|
|
1032
|
+
"map-rows": _map_rows,
|
|
1033
|
+
"redact-rows": _redact_rows,
|
|
1034
|
+
"redact-policy": _redact_policy,
|
|
981
1035
|
"conform": _conform,
|
|
982
1036
|
"reconcile": _reconcile,
|
|
983
1037
|
"run-tests": _run_tests,
|
|
@@ -20,6 +20,7 @@ from typing import Any, Callable
|
|
|
20
20
|
from ..legacy_sqlite import LegacyVerifiers
|
|
21
21
|
from .compiler import bundle_mode
|
|
22
22
|
from .data import QUERY_DEFAULT_LIMIT, entity_counts, entity_query, entity_query_across, entity_rows, world_kind
|
|
23
|
+
from .process_memory import SIZE, ProcessStatus
|
|
23
24
|
from .validation import ContractError, fail
|
|
24
25
|
from .world_sql import WorldSql
|
|
25
26
|
|
|
@@ -127,10 +128,10 @@ def cap_memory_for(request: Any, status: Path = Path("/proc/self/status"), limit
|
|
|
127
128
|
return
|
|
128
129
|
if limits is None:
|
|
129
130
|
import resource as limits
|
|
130
|
-
|
|
131
|
-
if
|
|
131
|
+
size = ProcessStatus.read(status).bytes(SIZE)
|
|
132
|
+
if size is None:
|
|
132
133
|
fail("runtime", "This process cannot read its own memory size, so it refuses to run caller SQL")
|
|
133
|
-
cap =
|
|
134
|
+
cap = size + SQL_MEMORY_BYTES
|
|
134
135
|
soft, hard = limits.getrlimit(limits.RLIMIT_AS)
|
|
135
136
|
hard = _lower(cap, hard, limits.RLIM_INFINITY)
|
|
136
137
|
limits.setrlimit(limits.RLIMIT_AS, (_lower(hard, soft, limits.RLIM_INFINITY), hard))
|
|
@@ -10,15 +10,19 @@ from __future__ import annotations
|
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
12
|
import tempfile
|
|
13
|
-
from
|
|
13
|
+
from dataclasses import replace
|
|
14
|
+
from pathlib import Path, PurePosixPath
|
|
14
15
|
from typing import TYPE_CHECKING, Any, Callable
|
|
15
16
|
|
|
16
17
|
from .cast import world_fields
|
|
18
|
+
from .shape_catalog import CatalogWorld, ShapeCatalog
|
|
19
|
+
from .shape_declarations import answer_declarations
|
|
17
20
|
from .scaffold import (
|
|
18
21
|
CUSTOM,
|
|
19
22
|
TEMPLATE_SLUG,
|
|
20
23
|
CustomScaffold,
|
|
21
24
|
TemplateWorld,
|
|
25
|
+
WorldTemplate,
|
|
22
26
|
init_bundle,
|
|
23
27
|
init_bundle_from_files,
|
|
24
28
|
init_bundle_from_world,
|
|
@@ -26,7 +30,7 @@ from .scaffold import (
|
|
|
26
30
|
request_files,
|
|
27
31
|
template_digest,
|
|
28
32
|
)
|
|
29
|
-
from .validation import ContractError, fail
|
|
33
|
+
from .validation import ContractError, fail, parse_toml
|
|
30
34
|
|
|
31
35
|
if TYPE_CHECKING:
|
|
32
36
|
from .connector import Spec
|
|
@@ -164,6 +168,12 @@ def _operations(spec: "Spec") -> list[dict]:
|
|
|
164
168
|
]
|
|
165
169
|
|
|
166
170
|
|
|
171
|
+
def _redacts(policy: "Spec | RedactPolicy | None") -> bool:
|
|
172
|
+
if policy is None:
|
|
173
|
+
return False
|
|
174
|
+
return bool(policy.hash_fields or policy.drop_fields or policy.preserve_fields or policy.round_fields)
|
|
175
|
+
|
|
176
|
+
|
|
167
177
|
def _served_operations(spec: "Spec", settings: Any) -> list[dict]:
|
|
168
178
|
operations = _operations(spec)
|
|
169
179
|
for operation in operations:
|
|
@@ -173,13 +183,13 @@ def _served_operations(spec: "Spec", settings: Any) -> list[dict]:
|
|
|
173
183
|
return operations
|
|
174
184
|
|
|
175
185
|
|
|
176
|
-
def template_summary(files: dict[str, bytes]) -> dict:
|
|
186
|
+
def template_summary(files: dict[str, bytes], resolved: CatalogWorld | ContractError | None = None) -> dict:
|
|
177
187
|
"""Never runs the template's Python."""
|
|
178
188
|
from .connector import bind_spec, parse_spec
|
|
179
189
|
from .redaction import parse_env_policy
|
|
180
190
|
|
|
181
|
-
world_path =
|
|
182
|
-
if world_path
|
|
191
|
+
world_path = "schema/world.json" if _world_tree(files) else "world.json"
|
|
192
|
+
if world_path not in files:
|
|
183
193
|
fail(
|
|
184
194
|
"template",
|
|
185
195
|
"No world.json (template) or schema/world.json (world) among the files",
|
|
@@ -196,7 +206,8 @@ def template_summary(files: dict[str, bytes]) -> dict:
|
|
|
196
206
|
if policy is None and "gateway-env.toml" in files:
|
|
197
207
|
policy = parse_env_policy(files["gateway-env.toml"].decode("utf-8"))
|
|
198
208
|
entities = []
|
|
199
|
-
|
|
209
|
+
resolved = resolved if resolved is not None else _resolved_files(files)
|
|
210
|
+
fields_by_entity = resolved.entity_fields() if isinstance(resolved, CatalogWorld) else world_fields(world)
|
|
200
211
|
for name, entity in sorted((gateway.get("entities") or {}).items()):
|
|
201
212
|
entities.append(
|
|
202
213
|
{
|
|
@@ -235,23 +246,26 @@ def template_summary(files: dict[str, bytes]) -> dict:
|
|
|
235
246
|
|
|
236
247
|
|
|
237
248
|
def describe(request: dict) -> dict:
|
|
238
|
-
"""``connector`` (``custom`` only) or the ``files`` of a template or a compiled world
|
|
249
|
+
"""``connector`` (``custom`` only) or the ``files`` of a template or a compiled world;
|
|
250
|
+
``entity`` narrows the description to that entity's fields and catalog entries."""
|
|
239
251
|
from .platform import describe_dependencies
|
|
252
|
+
from .shape_catalog import narrowed_description
|
|
240
253
|
from .ui import ui_kind
|
|
241
254
|
|
|
242
255
|
with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
|
|
243
256
|
root = Path(directory) / "bundle"
|
|
244
257
|
authored = _described_tree(request, root)
|
|
245
|
-
|
|
258
|
+
resolved = _resolved_world(root)
|
|
259
|
+
summary = template_summary({"world.json": (root / "schema/world.json").read_bytes()}, resolved)["summary"]
|
|
246
260
|
summary["digest"] = template_digest(authored)
|
|
247
261
|
summary["files"] = sorted(authored)
|
|
248
262
|
summary["ui"] = ui_kind(root)
|
|
249
263
|
summary["tools"] = _tool_schemas(root)
|
|
250
264
|
summary["connection"] = None
|
|
251
265
|
summary["openapi"] = None
|
|
252
|
-
_describe_surface(root, summary)
|
|
266
|
+
_describe_surface(root, summary, resolved)
|
|
253
267
|
summary["dependencies"] = describe_dependencies(root) if (root / "gateway-env.toml").is_file() else []
|
|
254
|
-
return {"status": "ok", "description": summary}
|
|
268
|
+
return {"status": "ok", "description": narrowed_description(summary, request.get("entity"))}
|
|
255
269
|
|
|
256
270
|
|
|
257
271
|
def _described_tree(request: dict, root: Path) -> dict[str, bytes]:
|
|
@@ -261,7 +275,7 @@ def _described_tree(request: dict, root: Path) -> dict[str, bytes]:
|
|
|
261
275
|
if connector is not None:
|
|
262
276
|
return _custom_tree(connector, root)
|
|
263
277
|
authored = request_files(request)
|
|
264
|
-
if
|
|
278
|
+
if _world_tree(authored):
|
|
265
279
|
_write_compiled(authored, root)
|
|
266
280
|
return authored
|
|
267
281
|
slug = request.get("slug") if isinstance(request.get("slug"), str) else None
|
|
@@ -282,13 +296,31 @@ def _custom_tree(connector: Any, root: Path) -> dict[str, bytes]:
|
|
|
282
296
|
|
|
283
297
|
def _write_compiled(authored: dict[str, bytes], root: Path) -> None:
|
|
284
298
|
root.mkdir()
|
|
299
|
+
laid_out = set(DESCRIBE_PATHS) | _settings_paths(authored.get("connector.toml"))
|
|
285
300
|
for path, data in authored.items():
|
|
286
|
-
if path in
|
|
301
|
+
if path in laid_out or _schema_sibling(path):
|
|
287
302
|
target = root / path
|
|
288
303
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
289
304
|
target.write_bytes(data)
|
|
290
305
|
|
|
291
306
|
|
|
307
|
+
def _schema_sibling(path: str) -> bool:
|
|
308
|
+
return path.startswith("schema/") and path.count("/") == 1 and path.endswith(".json")
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _settings_paths(spec: bytes | None) -> set[str]:
|
|
312
|
+
"""What ``load_api_settings`` looks for beside connector.toml: the handlers module it names and
|
|
313
|
+
the built ui's index, so a compiled world is described with the settings it is served with."""
|
|
314
|
+
from .api import FILTER_KEY, HANDLERS
|
|
315
|
+
|
|
316
|
+
raw = parse_toml(spec.decode("utf-8")) if spec is not None else {}
|
|
317
|
+
api = raw.get(FILTER_KEY) if isinstance(raw.get(FILTER_KEY), dict) else {}
|
|
318
|
+
ui = raw.get("ui") if isinstance(raw.get("ui"), dict) else {}
|
|
319
|
+
handlers = PurePosixPath(api.get("handlers") if isinstance(api.get("handlers"), str) else HANDLERS).as_posix()
|
|
320
|
+
static = ui.get("static")
|
|
321
|
+
return {handlers, (PurePosixPath(static) / "index.html").as_posix()} if isinstance(static, str) else {handlers}
|
|
322
|
+
|
|
323
|
+
|
|
292
324
|
def _tool_schemas(root: Path) -> list[dict]:
|
|
293
325
|
tool_schemas = {}
|
|
294
326
|
if (root / "tool-schemas.json").is_file():
|
|
@@ -304,7 +336,7 @@ def _tool_schemas(root: Path) -> list[dict]:
|
|
|
304
336
|
]
|
|
305
337
|
|
|
306
338
|
|
|
307
|
-
def _describe_surface(root: Path, summary: dict) -> None:
|
|
339
|
+
def _describe_surface(root: Path, summary: dict, resolved: CatalogWorld | ContractError) -> None:
|
|
308
340
|
from .api import load_api_settings, openapi_document
|
|
309
341
|
from .connector import load_spec
|
|
310
342
|
from .redaction import env_policy
|
|
@@ -313,14 +345,15 @@ def _describe_surface(root: Path, summary: dict) -> None:
|
|
|
313
345
|
policy = env_policy(root / "gateway-env.toml")
|
|
314
346
|
_annotate_redaction(summary["entities"], policy)
|
|
315
347
|
summary["redact"] = _redact_summary(policy) if policy else None
|
|
348
|
+
_catalog(summary, lambda: replace(_unrefused(resolved), redacted=_redacts(policy)))
|
|
316
349
|
return
|
|
317
350
|
spec = load_spec(root)
|
|
318
351
|
try:
|
|
319
|
-
settings = load_api_settings(root, spec)
|
|
320
|
-
except ContractError:
|
|
321
|
-
|
|
322
|
-
settings = None
|
|
352
|
+
settings, refused = load_api_settings(root, spec), None
|
|
353
|
+
except ContractError as error:
|
|
354
|
+
settings, refused = None, error
|
|
323
355
|
_annotate_redaction(summary["entities"], spec)
|
|
356
|
+
_catalog(summary, lambda: _served_world(root, spec, settings, refused, resolved))
|
|
324
357
|
summary["connection"] = {
|
|
325
358
|
"slug": spec.slug,
|
|
326
359
|
"baseUrl": spec.base_url,
|
|
@@ -330,3 +363,58 @@ def _describe_surface(root: Path, summary: dict) -> None:
|
|
|
330
363
|
}
|
|
331
364
|
summary["openapi"] = openapi_document(spec, summary.get("domainId") or spec.slug, settings)
|
|
332
365
|
summary["redact"] = summary["connection"]["redact"]
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _served_world(
|
|
369
|
+
root: Path, spec: "Spec", settings: Any, refused: ContractError | None, resolved: CatalogWorld | ContractError
|
|
370
|
+
) -> CatalogWorld:
|
|
371
|
+
if refused is not None:
|
|
372
|
+
raise refused
|
|
373
|
+
answers = answer_declarations(parse_toml((root / "connector.toml").read_text(encoding="utf-8")))
|
|
374
|
+
return replace(_unrefused(resolved), spec=spec, settings=settings, answers=answers, redacted=_redacts(spec))
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _resolved_world(root: Path) -> CatalogWorld | ContractError:
|
|
378
|
+
try:
|
|
379
|
+
return CatalogWorld.of_bundle(root)
|
|
380
|
+
except ContractError as refused:
|
|
381
|
+
return refused
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _unrefused(resolved: CatalogWorld | ContractError) -> CatalogWorld:
|
|
385
|
+
if isinstance(resolved, ContractError):
|
|
386
|
+
raise resolved
|
|
387
|
+
return resolved
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _resolved_files(files: dict[str, bytes]) -> CatalogWorld | ContractError:
|
|
391
|
+
with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
|
|
392
|
+
root = Path(directory) / "bundle"
|
|
393
|
+
try:
|
|
394
|
+
_lay_out_schema(files, root)
|
|
395
|
+
except ContractError as refused:
|
|
396
|
+
return refused
|
|
397
|
+
return _resolved_world(root)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _lay_out_schema(files: dict[str, bytes], root: Path) -> None:
|
|
401
|
+
if _world_tree(files):
|
|
402
|
+
_write_compiled(files, root)
|
|
403
|
+
return
|
|
404
|
+
WorldTemplate(files).lay_out_schema(root)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _world_tree(files: dict[str, bytes]) -> bool:
|
|
408
|
+
"""A built world's files, whole or only their contract half (no ``tool-schemas.json``); else a template's."""
|
|
409
|
+
compiled = all(marker in files for marker in COMPILED_MARKERS)
|
|
410
|
+
return compiled or ("schema/world.json" in files and "world.json" not in files)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _catalog(summary: dict, world: Callable[[], CatalogWorld]) -> None:
|
|
414
|
+
"""The catalog describes the world as it is served. When that cannot be read the description
|
|
415
|
+
keeps its entities and says why under ``catalogRefused``, rather than answer an empty catalog."""
|
|
416
|
+
try:
|
|
417
|
+
summary["catalog"] = ShapeCatalog.of(world()).document()
|
|
418
|
+
except ContractError as error:
|
|
419
|
+
summary["catalog"] = None
|
|
420
|
+
summary["catalogRefused"] = error.detail
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""A process's memory as its kernel counts it: the ``Vm*`` and ``Rss*`` lines of ``/proc/<pid>/status``,
|
|
2
|
+
and a reaped child's ``ru_maxrss``.
|
|
3
|
+
|
|
4
|
+
Read from the kernel, never asked of the process, so a worker running tenant code cannot
|
|
5
|
+
under-report what it holds. Where there is no such file (no ``/proc``, as on macOS, or a
|
|
6
|
+
process already gone) every figure is ``None`` and the caller falls back to what it was told.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
import sys
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
SIZE = "VmSize"
|
|
17
|
+
PEAK_SIZE = "VmPeak"
|
|
18
|
+
RESIDENT = "VmRSS"
|
|
19
|
+
RESIDENT_ANON = "RssAnon"
|
|
20
|
+
PEAK_RESIDENT = "VmHWM"
|
|
21
|
+
#: What a world's measured peak is multiplied by for what it needs of a worker (``conformance/world-sessions.json``).
|
|
22
|
+
HEADROOM = 1.5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class ProcessStatus:
|
|
27
|
+
fields: dict[str, int] = field(default_factory=dict)
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def read(cls, path: Path) -> ProcessStatus:
|
|
31
|
+
try:
|
|
32
|
+
text = path.read_text()
|
|
33
|
+
except OSError:
|
|
34
|
+
return cls()
|
|
35
|
+
return cls(dict(reading for line in text.splitlines() if (reading := _kib_line(line)) is not None))
|
|
36
|
+
|
|
37
|
+
def bytes(self, name: str) -> int | None:
|
|
38
|
+
return self.fields.get(name)
|
|
39
|
+
|
|
40
|
+
def mb(self, name: str) -> int | None:
|
|
41
|
+
value = self.fields.get(name)
|
|
42
|
+
return None if value is None else value >> 20
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class MemoryNeed:
|
|
47
|
+
"""What a world needs of a worker: its measured peak and half again."""
|
|
48
|
+
|
|
49
|
+
peak_mb: int
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def mb(self) -> int:
|
|
53
|
+
return math.ceil(self.peak_mb * HEADROOM)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ProcessMemory:
|
|
57
|
+
"""Where ``/proc`` is read; a test hands it a directory laid out the same way."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, proc: Path = Path("/proc")) -> None:
|
|
60
|
+
self.proc = proc
|
|
61
|
+
|
|
62
|
+
def status(self, pid: int | str = "self") -> ProcessStatus:
|
|
63
|
+
return ProcessStatus.read(self.proc / str(pid) / "status")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def maxrss_mb(maxrss: int) -> int:
|
|
67
|
+
"""A ``ru_maxrss`` the kernel reported (KiB on Linux, bytes on macOS) in MiB."""
|
|
68
|
+
return maxrss >> 20 if sys.platform == "darwin" else maxrss >> 10
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _kib_line(line: str) -> tuple[str, int] | None:
|
|
72
|
+
name, _, rest = line.partition(":")
|
|
73
|
+
parts = rest.split()
|
|
74
|
+
if not name.startswith(("Vm", "Rss")) or len(parts) != 2 or parts[1] != "kB" or not parts[0].isdigit():
|
|
75
|
+
return None
|
|
76
|
+
return name, int(parts[0]) << 10
|
|
@@ -82,6 +82,19 @@ def policy_for(root: str | Path) -> Any | None:
|
|
|
82
82
|
return env_policy(Path(root) / ENV_FILE)
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
def policy_view(spec: Any | None) -> dict | None:
|
|
86
|
+
"""What a policy does to rows, as plain JSON, so two versions of a world can be compared: the salt and each field list."""
|
|
87
|
+
if spec is None:
|
|
88
|
+
return None
|
|
89
|
+
return {
|
|
90
|
+
"salt": spec.slug,
|
|
91
|
+
"hash": sorted(spec.hash_fields),
|
|
92
|
+
"drop": sorted(spec.drop_fields),
|
|
93
|
+
"round": dict(sorted(spec.round_fields.items())),
|
|
94
|
+
"preserve": sorted(spec.preserve_fields),
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
85
98
|
def env_policy(path: Path) -> RedactPolicy | None:
|
|
86
99
|
"""``[redact]`` out of a ``gateway-env.toml``, or None when the file or the table is absent."""
|
|
87
100
|
if not path.is_file():
|
|
@@ -77,6 +77,11 @@ class WorldRuntime(ABC):
|
|
|
77
77
|
def authored_code(cls, cached: CachedVersion) -> str | None:
|
|
78
78
|
"""World-authored code ``cached`` runs, so only a version worker may open it; None when it runs none."""
|
|
79
79
|
|
|
80
|
+
@classmethod
|
|
81
|
+
@abstractmethod
|
|
82
|
+
def world_name(cls, cached: CachedVersion) -> str:
|
|
83
|
+
"""The name ``cached`` gives its world, for what the host logs about it."""
|
|
84
|
+
|
|
80
85
|
@classmethod
|
|
81
86
|
def declared_dependencies(cls, cached: CachedVersion) -> dict[str, Dependency]:
|
|
82
87
|
return {}
|
|
@@ -375,6 +380,11 @@ class TreeRuntime(WorldRuntime):
|
|
|
375
380
|
def authored_code(cls, cached: CachedVersion) -> str:
|
|
376
381
|
return "schema world handlers"
|
|
377
382
|
|
|
383
|
+
@classmethod
|
|
384
|
+
def world_name(cls, cached: CachedVersion) -> str:
|
|
385
|
+
assert cached.tools is not None
|
|
386
|
+
return cached.tools.manifest.name
|
|
387
|
+
|
|
378
388
|
@classmethod
|
|
379
389
|
def declared_dependencies(cls, cached: CachedVersion) -> dict[str, Dependency]:
|
|
380
390
|
assert cached.tools is not None
|
|
@@ -522,6 +532,10 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
522
532
|
return None
|
|
523
533
|
return f"api handlers module {cached.settings.handlers.name}"
|
|
524
534
|
|
|
535
|
+
@classmethod
|
|
536
|
+
def world_name(cls, cached: CachedVersion) -> str:
|
|
537
|
+
return cached.spec.slug if cached.spec is not None else cached.version_id
|
|
538
|
+
|
|
525
539
|
@staticmethod
|
|
526
540
|
def live_state(session_dir: Path) -> Path:
|
|
527
541
|
return HostWorldState.live_state(session_dir)
|