@withgateway/sdk 0.6.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +581 -0
- package/dist/agents/index.d.ts +581 -0
- package/dist/agents/index.js +1473 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1424 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +11611 -6845
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +10090 -5325
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +3177 -748
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2987 -590
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1986 -1746
- package/dist/worlds/index.d.ts +1986 -1746
- package/dist/worlds/index.js +1470 -290
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1465 -286
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +94 -12
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
- package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +2 -2
- package/skills/connector-schema-authoring/connector-worlds.md +3 -2
- package/skills/connector-schema-authoring/world-schema.md +36 -23
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +8 -6
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
"""The bridge's template commands: ``templates``, ``template-summary``, ``describe``,
|
|
2
|
+
``init-template``, ``init-template-files``, ``init-world-files``, ``build-slack`` and ``refresh-slack``.
|
|
3
|
+
|
|
4
|
+
A vendor template is an official world on the platform, a schema world built from it, and
|
|
5
|
+
arrives as that world's files (``[{path, content}]``, base64); only the blank ``custom``
|
|
6
|
+
scaffold is read from code. None of these runs a template's Python.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import tempfile
|
|
13
|
+
from dataclasses import replace
|
|
14
|
+
from pathlib import Path, PurePosixPath
|
|
15
|
+
from typing import TYPE_CHECKING, Any, Callable
|
|
16
|
+
|
|
17
|
+
from .cast import world_fields
|
|
18
|
+
from .shape_catalog import CatalogWorld, ShapeCatalog
|
|
19
|
+
from .shape_declarations import answer_declarations
|
|
20
|
+
from .scaffold import (
|
|
21
|
+
CUSTOM,
|
|
22
|
+
TEMPLATE_SLUG,
|
|
23
|
+
CustomScaffold,
|
|
24
|
+
TemplateWorld,
|
|
25
|
+
WorldTemplate,
|
|
26
|
+
init_bundle,
|
|
27
|
+
init_bundle_from_files,
|
|
28
|
+
init_bundle_from_world,
|
|
29
|
+
refuse_official,
|
|
30
|
+
request_files,
|
|
31
|
+
template_digest,
|
|
32
|
+
)
|
|
33
|
+
from .validation import ContractError, fail, parse_toml
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from .connector import Spec
|
|
37
|
+
from .redaction import RedactPolicy
|
|
38
|
+
|
|
39
|
+
COMPILED_MARKERS = ("schema/world.json", "tool-schemas.json")
|
|
40
|
+
DESCRIBE_PATHS = (
|
|
41
|
+
"schema/world.json",
|
|
42
|
+
"tool-schemas.json",
|
|
43
|
+
"connector.toml",
|
|
44
|
+
"gateway-env.toml",
|
|
45
|
+
"api/handlers.py",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def templates(request: dict) -> dict:
|
|
50
|
+
"""A summary of each official world the caller sends, in its order, then ``custom``. A world whose
|
|
51
|
+
own files cannot be summarized is answered under ``refused`` and the others still list; a
|
|
52
|
+
request of the wrong shape is refused whole."""
|
|
53
|
+
listed = request.get("templates")
|
|
54
|
+
listed = [] if listed is None else listed
|
|
55
|
+
if not isinstance(listed, list):
|
|
56
|
+
fail("input", "Expected templates as [{slug, files: [{path, content}]}]", "/templates")
|
|
57
|
+
summaries: list[dict] = []
|
|
58
|
+
refused: list[dict] = []
|
|
59
|
+
seen: set[str] = set()
|
|
60
|
+
for index, entry in enumerate(listed):
|
|
61
|
+
location = f"/templates/{index}"
|
|
62
|
+
slug = _official_slug(entry, location)
|
|
63
|
+
if slug in seen:
|
|
64
|
+
fail("input", f"{slug} is sent twice", f"{location}/slug")
|
|
65
|
+
seen.add(slug)
|
|
66
|
+
template = TemplateWorld(request_files(entry, location=f"{location}/files"), location=f"{location}/files")
|
|
67
|
+
try:
|
|
68
|
+
summaries.append(template.summary(slug))
|
|
69
|
+
except ContractError as error:
|
|
70
|
+
refused.append({"slug": slug, "error": error.detail})
|
|
71
|
+
return {"status": "ok", "templates": [*summaries, CustomScaffold().summary(CUSTOM)], "refused": refused}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _official_slug(entry: Any, location: str) -> str:
|
|
75
|
+
slug = entry.get("slug") if isinstance(entry, dict) else None
|
|
76
|
+
if not isinstance(slug, str) or not TEMPLATE_SLUG.fullmatch(slug):
|
|
77
|
+
fail(
|
|
78
|
+
"input", "Each template is {slug, files} with a slug of lowercase letters, digits and -", f"{location}/slug"
|
|
79
|
+
)
|
|
80
|
+
if slug == CUSTOM:
|
|
81
|
+
fail(
|
|
82
|
+
"input",
|
|
83
|
+
"custom is the runtime's own scaffold and is always listed last; send official templates only",
|
|
84
|
+
f"{location}/slug",
|
|
85
|
+
)
|
|
86
|
+
return slug
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def init_template(request: dict, root: Path) -> None:
|
|
90
|
+
connector = request.get("connector", CUSTOM)
|
|
91
|
+
name = request.get("name")
|
|
92
|
+
if not isinstance(connector, str) or (name is not None and not isinstance(name, str)):
|
|
93
|
+
fail("input", "Expected a connector template slug and an optional name")
|
|
94
|
+
refuse_official(connector, "init-template-files")
|
|
95
|
+
init_bundle(root, connector=CUSTOM, name=name, with_example_tool=request.get("with_example_tool") is True)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def init_template_files(request: dict, root: Path) -> None:
|
|
99
|
+
"""The template arrives as files, never as a path the request chose."""
|
|
100
|
+
connector = request.get("connector")
|
|
101
|
+
name = request.get("name")
|
|
102
|
+
if not isinstance(connector, str) or (name is not None and not isinstance(name, str)):
|
|
103
|
+
fail("input", "Expected the connector slug and an optional name")
|
|
104
|
+
init_bundle_from_files(root, request_files(request), connector=connector, name=name)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def init_world_files(request: dict, root: Path) -> None:
|
|
108
|
+
"""An official world's files, copied under ``name`` and compiled."""
|
|
109
|
+
name = request.get("name")
|
|
110
|
+
if not isinstance(name, str):
|
|
111
|
+
fail("input", "Expected the new world's name")
|
|
112
|
+
init_bundle_from_world(root, request_files(request), name=name)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def build_slack(request: dict, root: Path) -> None:
|
|
116
|
+
from .slack import build_slack_bundle
|
|
117
|
+
|
|
118
|
+
template = request_files(request, "template")
|
|
119
|
+
build_slack_bundle(root, template=template, evidence=request.get("evidence"), name=request.get("name"))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def refresh_slack(request: dict, root: Path) -> None:
|
|
123
|
+
from .slack import refresh_slack_bundle
|
|
124
|
+
|
|
125
|
+
template = request_files(request, "template")
|
|
126
|
+
refresh_slack_bundle(root, request.get("evidence"), template=template, batch_id=request.get("batch_id"))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
WRITERS: dict[str, Callable[[dict, Path], None]] = {
|
|
130
|
+
"init-template": init_template,
|
|
131
|
+
"init-template-files": init_template_files,
|
|
132
|
+
"init-world-files": init_world_files,
|
|
133
|
+
"build-slack": build_slack,
|
|
134
|
+
"refresh-slack": refresh_slack,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _redact_summary(spec: "Spec | RedactPolicy") -> dict:
|
|
139
|
+
"""``slug`` is the salt ``_digest`` hashes under: the connector's slug, not the platform world slug."""
|
|
140
|
+
return {
|
|
141
|
+
"slug": spec.slug,
|
|
142
|
+
"hash": sorted(spec.hash_fields),
|
|
143
|
+
"drop": sorted(spec.drop_fields),
|
|
144
|
+
"preserve": sorted(spec.preserve_fields),
|
|
145
|
+
"round": dict(spec.round_fields),
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _annotate_redaction(entities: list[dict], spec: "Spec | RedactPolicy | None") -> None:
|
|
150
|
+
from .connector import redaction_of
|
|
151
|
+
|
|
152
|
+
for entity in entities:
|
|
153
|
+
for field in entity["fields"]:
|
|
154
|
+
field["redaction"] = redaction_of(spec, field["name"]) if spec else None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _operations(spec: "Spec") -> list[dict]:
|
|
158
|
+
return [
|
|
159
|
+
{
|
|
160
|
+
"name": op.name,
|
|
161
|
+
"method": op.method,
|
|
162
|
+
"path": op.path,
|
|
163
|
+
"params": list(op.params),
|
|
164
|
+
"entity": op.entity,
|
|
165
|
+
"results": op.results,
|
|
166
|
+
}
|
|
167
|
+
for op in spec.operations.values()
|
|
168
|
+
]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _redacts(policy: "Spec | RedactPolicy | None") -> bool:
|
|
172
|
+
if policy is None:
|
|
173
|
+
return False
|
|
174
|
+
return bool(policy.hash_fields or policy.drop_fields or policy.preserve_fields or policy.round_fields)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _served_operations(spec: "Spec", settings: Any) -> list[dict]:
|
|
178
|
+
operations = _operations(spec)
|
|
179
|
+
for operation in operations:
|
|
180
|
+
served = settings.for_operation(operation["name"]) if settings else None
|
|
181
|
+
operation["action"] = served.action if served else None
|
|
182
|
+
operation["status"] = served.status if served else None
|
|
183
|
+
return operations
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def template_summary(files: dict[str, bytes], resolved: CatalogWorld | ContractError | None = None) -> dict:
|
|
187
|
+
"""Never runs the template's Python."""
|
|
188
|
+
from .connector import bind_spec, parse_spec
|
|
189
|
+
from .redaction import parse_env_policy
|
|
190
|
+
|
|
191
|
+
world_path = "schema/world.json" if _world_tree(files) else "world.json"
|
|
192
|
+
if world_path not in files:
|
|
193
|
+
fail(
|
|
194
|
+
"template",
|
|
195
|
+
"No world.json (template) or schema/world.json (world) among the files",
|
|
196
|
+
)
|
|
197
|
+
try:
|
|
198
|
+
world = json.loads(files[world_path].decode("utf-8"))
|
|
199
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
200
|
+
fail("template", f"{world_path} is not valid JSON: {error}")
|
|
201
|
+
gateway = world.get("x-gateway") if isinstance(world, dict) else None
|
|
202
|
+
if not isinstance(gateway, dict):
|
|
203
|
+
fail("template", f"{world_path} is not a gateway-world document")
|
|
204
|
+
spec = bind_spec(parse_spec(files["connector.toml"].decode("utf-8")), world) if "connector.toml" in files else None
|
|
205
|
+
policy = spec
|
|
206
|
+
if policy is None and "gateway-env.toml" in files:
|
|
207
|
+
policy = parse_env_policy(files["gateway-env.toml"].decode("utf-8"))
|
|
208
|
+
entities = []
|
|
209
|
+
resolved = resolved if resolved is not None else _resolved_files(files)
|
|
210
|
+
fields_by_entity = resolved.entity_fields() if isinstance(resolved, CatalogWorld) else world_fields(world)
|
|
211
|
+
for name, entity in sorted((gateway.get("entities") or {}).items()):
|
|
212
|
+
entities.append(
|
|
213
|
+
{
|
|
214
|
+
"name": name,
|
|
215
|
+
"primaryKey": list(entity.get("primary_key") or []) if isinstance(entity, dict) else [],
|
|
216
|
+
"unique": list(entity.get("unique") or []) if isinstance(entity, dict) else [],
|
|
217
|
+
"relationships": list(entity.get("relationships") or []) if isinstance(entity, dict) else [],
|
|
218
|
+
"fields": [field.describe() for field in fields_by_entity.get(name, [])],
|
|
219
|
+
}
|
|
220
|
+
)
|
|
221
|
+
_annotate_redaction(entities, policy)
|
|
222
|
+
connection = None
|
|
223
|
+
if spec is not None:
|
|
224
|
+
connection = {
|
|
225
|
+
"slug": spec.slug,
|
|
226
|
+
"baseUrl": spec.base_url,
|
|
227
|
+
"auth": spec.scheme,
|
|
228
|
+
"redact": _redact_summary(spec),
|
|
229
|
+
"operations": _operations(spec),
|
|
230
|
+
}
|
|
231
|
+
package = gateway.get("package") or {}
|
|
232
|
+
return {
|
|
233
|
+
"status": "ok",
|
|
234
|
+
"summary": {
|
|
235
|
+
"packageVersion": package.get("version"),
|
|
236
|
+
"domainVersion": gateway.get("domain_version"),
|
|
237
|
+
"domainId": world.get("$id"),
|
|
238
|
+
"digest": template_digest(files),
|
|
239
|
+
"entities": entities,
|
|
240
|
+
"connection": connection,
|
|
241
|
+
"redact": _redact_summary(policy) if policy is not None else None,
|
|
242
|
+
"tools": sorted((gateway.get("tools") or {}).keys()),
|
|
243
|
+
"files": sorted(files),
|
|
244
|
+
},
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def describe(request: dict) -> dict:
|
|
249
|
+
"""``connector`` (``custom`` only) or the ``files`` of a template or a compiled world;
|
|
250
|
+
``entity`` narrows the description to that entity's fields and catalog entries."""
|
|
251
|
+
from .platform import describe_dependencies
|
|
252
|
+
from .shape_catalog import narrowed_description
|
|
253
|
+
from .ui import ui_kind
|
|
254
|
+
|
|
255
|
+
with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
|
|
256
|
+
root = Path(directory) / "bundle"
|
|
257
|
+
authored = _described_tree(request, root)
|
|
258
|
+
resolved = _resolved_world(root)
|
|
259
|
+
summary = template_summary({"world.json": (root / "schema/world.json").read_bytes()}, resolved)["summary"]
|
|
260
|
+
summary["digest"] = template_digest(authored)
|
|
261
|
+
summary["files"] = sorted(authored)
|
|
262
|
+
summary["ui"] = ui_kind(root)
|
|
263
|
+
summary["tools"] = _tool_schemas(root)
|
|
264
|
+
summary["connection"] = None
|
|
265
|
+
summary["openapi"] = None
|
|
266
|
+
_describe_surface(root, summary, resolved)
|
|
267
|
+
summary["dependencies"] = describe_dependencies(root) if (root / "gateway-env.toml").is_file() else []
|
|
268
|
+
return {"status": "ok", "description": narrowed_description(summary, request.get("entity"))}
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _described_tree(request: dict, root: Path) -> dict[str, bytes]:
|
|
272
|
+
from .connector import parse_spec
|
|
273
|
+
|
|
274
|
+
connector = request.get("connector")
|
|
275
|
+
if connector is not None:
|
|
276
|
+
return _custom_tree(connector, root)
|
|
277
|
+
authored = request_files(request)
|
|
278
|
+
if _world_tree(authored):
|
|
279
|
+
_write_compiled(authored, root)
|
|
280
|
+
return authored
|
|
281
|
+
slug = request.get("slug") if isinstance(request.get("slug"), str) else None
|
|
282
|
+
if slug is None and "connector.toml" in authored:
|
|
283
|
+
slug = parse_spec(authored["connector.toml"].decode("utf-8")).slug
|
|
284
|
+
init_bundle_from_files(root, authored, connector=slug or CUSTOM)
|
|
285
|
+
return authored
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _custom_tree(connector: Any, root: Path) -> dict[str, bytes]:
|
|
289
|
+
if not isinstance(connector, str):
|
|
290
|
+
fail("input", "Expected a connector template slug", "/connector")
|
|
291
|
+
refuse_official(connector, "describe", "/connector")
|
|
292
|
+
scaffold = CustomScaffold()
|
|
293
|
+
scaffold.write(root, connector=CUSTOM)
|
|
294
|
+
return scaffold.files
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _write_compiled(authored: dict[str, bytes], root: Path) -> None:
|
|
298
|
+
root.mkdir()
|
|
299
|
+
laid_out = set(DESCRIBE_PATHS) | _settings_paths(authored.get("connector.toml"))
|
|
300
|
+
for path, data in authored.items():
|
|
301
|
+
if path in laid_out or _schema_sibling(path):
|
|
302
|
+
target = root / path
|
|
303
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
304
|
+
target.write_bytes(data)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _schema_sibling(path: str) -> bool:
|
|
308
|
+
return path.startswith("schema/") and path.count("/") == 1 and path.endswith(".json")
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _settings_paths(spec: bytes | None) -> set[str]:
|
|
312
|
+
"""What ``load_api_settings`` looks for beside connector.toml: the handlers module it names and
|
|
313
|
+
the built ui's index, so a compiled world is described with the settings it is served with."""
|
|
314
|
+
from .api import FILTER_KEY, HANDLERS
|
|
315
|
+
|
|
316
|
+
raw = parse_toml(spec.decode("utf-8")) if spec is not None else {}
|
|
317
|
+
api = raw.get(FILTER_KEY) if isinstance(raw.get(FILTER_KEY), dict) else {}
|
|
318
|
+
ui = raw.get("ui") if isinstance(raw.get("ui"), dict) else {}
|
|
319
|
+
handlers = PurePosixPath(api.get("handlers") if isinstance(api.get("handlers"), str) else HANDLERS).as_posix()
|
|
320
|
+
static = ui.get("static")
|
|
321
|
+
return {handlers, (PurePosixPath(static) / "index.html").as_posix()} if isinstance(static, str) else {handlers}
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _tool_schemas(root: Path) -> list[dict]:
|
|
325
|
+
tool_schemas = {}
|
|
326
|
+
if (root / "tool-schemas.json").is_file():
|
|
327
|
+
tool_schemas = json.loads((root / "tool-schemas.json").read_text(encoding="utf-8")).get("tools") or {}
|
|
328
|
+
return [
|
|
329
|
+
{
|
|
330
|
+
"name": name,
|
|
331
|
+
"description": str(entry.get("description") or ""),
|
|
332
|
+
"inputSchema": entry.get("input_schema"),
|
|
333
|
+
"outputSchema": entry.get("output_schema"),
|
|
334
|
+
}
|
|
335
|
+
for name, entry in sorted(tool_schemas.items())
|
|
336
|
+
]
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _describe_surface(root: Path, summary: dict, resolved: CatalogWorld | ContractError) -> None:
|
|
340
|
+
from .api import load_api_settings, openapi_document
|
|
341
|
+
from .connector import load_spec
|
|
342
|
+
from .redaction import env_policy
|
|
343
|
+
|
|
344
|
+
if not (root / "connector.toml").is_file():
|
|
345
|
+
policy = env_policy(root / "gateway-env.toml")
|
|
346
|
+
_annotate_redaction(summary["entities"], policy)
|
|
347
|
+
summary["redact"] = _redact_summary(policy) if policy else None
|
|
348
|
+
_catalog(summary, lambda: replace(_unrefused(resolved), redacted=_redacts(policy)))
|
|
349
|
+
return
|
|
350
|
+
spec = load_spec(root)
|
|
351
|
+
try:
|
|
352
|
+
settings, refused = load_api_settings(root, spec), None
|
|
353
|
+
except ContractError as error:
|
|
354
|
+
settings, refused = None, error
|
|
355
|
+
_annotate_redaction(summary["entities"], spec)
|
|
356
|
+
_catalog(summary, lambda: _served_world(root, spec, settings, refused, resolved))
|
|
357
|
+
summary["connection"] = {
|
|
358
|
+
"slug": spec.slug,
|
|
359
|
+
"baseUrl": spec.base_url,
|
|
360
|
+
"auth": spec.scheme,
|
|
361
|
+
"redact": _redact_summary(spec),
|
|
362
|
+
"operations": _served_operations(spec, settings),
|
|
363
|
+
}
|
|
364
|
+
summary["openapi"] = openapi_document(spec, summary.get("domainId") or spec.slug, settings)
|
|
365
|
+
summary["redact"] = summary["connection"]["redact"]
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _served_world(
|
|
369
|
+
root: Path, spec: "Spec", settings: Any, refused: ContractError | None, resolved: CatalogWorld | ContractError
|
|
370
|
+
) -> CatalogWorld:
|
|
371
|
+
if refused is not None:
|
|
372
|
+
raise refused
|
|
373
|
+
answers = answer_declarations(parse_toml((root / "connector.toml").read_text(encoding="utf-8")))
|
|
374
|
+
return replace(_unrefused(resolved), spec=spec, settings=settings, answers=answers, redacted=_redacts(spec))
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _resolved_world(root: Path) -> CatalogWorld | ContractError:
|
|
378
|
+
try:
|
|
379
|
+
return CatalogWorld.of_bundle(root)
|
|
380
|
+
except ContractError as refused:
|
|
381
|
+
return refused
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _unrefused(resolved: CatalogWorld | ContractError) -> CatalogWorld:
|
|
385
|
+
if isinstance(resolved, ContractError):
|
|
386
|
+
raise resolved
|
|
387
|
+
return resolved
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _resolved_files(files: dict[str, bytes]) -> CatalogWorld | ContractError:
|
|
391
|
+
with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
|
|
392
|
+
root = Path(directory) / "bundle"
|
|
393
|
+
try:
|
|
394
|
+
_lay_out_schema(files, root)
|
|
395
|
+
except ContractError as refused:
|
|
396
|
+
return refused
|
|
397
|
+
return _resolved_world(root)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _lay_out_schema(files: dict[str, bytes], root: Path) -> None:
|
|
401
|
+
if _world_tree(files):
|
|
402
|
+
_write_compiled(files, root)
|
|
403
|
+
return
|
|
404
|
+
WorldTemplate(files).lay_out_schema(root)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _world_tree(files: dict[str, bytes]) -> bool:
|
|
408
|
+
"""A built world's files, whole or only their contract half (no ``tool-schemas.json``); else a template's."""
|
|
409
|
+
compiled = all(marker in files for marker in COMPILED_MARKERS)
|
|
410
|
+
return compiled or ("schema/world.json" in files and "world.json" not in files)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _catalog(summary: dict, world: Callable[[], CatalogWorld]) -> None:
|
|
414
|
+
"""The catalog describes the world as it is served. When that cannot be read the description
|
|
415
|
+
keeps its entities and says why under ``catalogRefused``, rather than answer an empty catalog."""
|
|
416
|
+
try:
|
|
417
|
+
summary["catalog"] = ShapeCatalog.of(world()).document()
|
|
418
|
+
except ContractError as error:
|
|
419
|
+
summary["catalog"] = None
|
|
420
|
+
summary["catalogRefused"] = error.detail
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""A process's memory as its kernel counts it: the ``Vm*`` and ``Rss*`` lines of ``/proc/<pid>/status``,
|
|
2
|
+
and a reaped child's ``ru_maxrss``.
|
|
3
|
+
|
|
4
|
+
Read from the kernel, never asked of the process, so a worker running tenant code cannot
|
|
5
|
+
under-report what it holds. Where there is no such file (no ``/proc``, as on macOS, or a
|
|
6
|
+
process already gone) every figure is ``None`` and the caller falls back to what it was told.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
import sys
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
SIZE = "VmSize"
|
|
17
|
+
PEAK_SIZE = "VmPeak"
|
|
18
|
+
RESIDENT = "VmRSS"
|
|
19
|
+
RESIDENT_ANON = "RssAnon"
|
|
20
|
+
PEAK_RESIDENT = "VmHWM"
|
|
21
|
+
#: What a world's measured peak is multiplied by for what it needs of a worker (``conformance/world-sessions.json``).
|
|
22
|
+
HEADROOM = 1.5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class ProcessStatus:
|
|
27
|
+
fields: dict[str, int] = field(default_factory=dict)
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def read(cls, path: Path) -> ProcessStatus:
|
|
31
|
+
try:
|
|
32
|
+
text = path.read_text()
|
|
33
|
+
except OSError:
|
|
34
|
+
return cls()
|
|
35
|
+
return cls(dict(reading for line in text.splitlines() if (reading := _kib_line(line)) is not None))
|
|
36
|
+
|
|
37
|
+
def bytes(self, name: str) -> int | None:
|
|
38
|
+
return self.fields.get(name)
|
|
39
|
+
|
|
40
|
+
def mb(self, name: str) -> int | None:
|
|
41
|
+
value = self.fields.get(name)
|
|
42
|
+
return None if value is None else value >> 20
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class MemoryNeed:
|
|
47
|
+
"""What a world needs of a worker: its measured peak and half again."""
|
|
48
|
+
|
|
49
|
+
peak_mb: int
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def mb(self) -> int:
|
|
53
|
+
return math.ceil(self.peak_mb * HEADROOM)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ProcessMemory:
|
|
57
|
+
"""Where ``/proc`` is read; a test hands it a directory laid out the same way."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, proc: Path = Path("/proc")) -> None:
|
|
60
|
+
self.proc = proc
|
|
61
|
+
|
|
62
|
+
def status(self, pid: int | str = "self") -> ProcessStatus:
|
|
63
|
+
return ProcessStatus.read(self.proc / str(pid) / "status")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def maxrss_mb(maxrss: int) -> int:
|
|
67
|
+
"""A ``ru_maxrss`` the kernel reported (KiB on Linux, bytes on macOS) in MiB."""
|
|
68
|
+
return maxrss >> 20 if sys.platform == "darwin" else maxrss >> 10
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _kib_line(line: str) -> tuple[str, int] | None:
|
|
72
|
+
name, _, rest = line.partition(":")
|
|
73
|
+
parts = rest.split()
|
|
74
|
+
if not name.startswith(("Vm", "Rss")) or len(parts) != 2 or parts[1] != "kB" or not parts[0].isdigit():
|
|
75
|
+
return None
|
|
76
|
+
return name, int(parts[0]) << 10
|
|
@@ -82,6 +82,19 @@ def policy_for(root: str | Path) -> Any | None:
|
|
|
82
82
|
return env_policy(Path(root) / ENV_FILE)
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
def policy_view(spec: Any | None) -> dict | None:
|
|
86
|
+
"""What a policy does to rows, as plain JSON, so two versions of a world can be compared: the salt and each field list."""
|
|
87
|
+
if spec is None:
|
|
88
|
+
return None
|
|
89
|
+
return {
|
|
90
|
+
"salt": spec.slug,
|
|
91
|
+
"hash": sorted(spec.hash_fields),
|
|
92
|
+
"drop": sorted(spec.drop_fields),
|
|
93
|
+
"round": dict(sorted(spec.round_fields.items())),
|
|
94
|
+
"preserve": sorted(spec.preserve_fields),
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
85
98
|
def env_policy(path: Path) -> RedactPolicy | None:
|
|
86
99
|
"""``[redact]`` out of a ``gateway-env.toml``, or None when the file or the table is absent."""
|
|
87
100
|
if not path.is_file():
|
|
@@ -434,11 +434,14 @@ class _Unredirected(urllib.request.HTTPRedirectHandler):
|
|
|
434
434
|
return None
|
|
435
435
|
|
|
436
436
|
|
|
437
|
+
UNREDIRECTED = urllib.request.build_opener(_Unredirected)
|
|
438
|
+
|
|
439
|
+
|
|
437
440
|
class PlatformLedger:
|
|
438
441
|
"""The platform's ledger door (:data:`LEDGER_ROUTE`), behind the control-plane secret; an answer that
|
|
439
442
|
redirects the flush is not followed, so it reads as unavailable, never as stored."""
|
|
440
443
|
|
|
441
|
-
_opener =
|
|
444
|
+
_opener = UNREDIRECTED
|
|
442
445
|
|
|
443
446
|
def __init__(self, platform_url: str, secret: str, timeout_s: float = 30.0, host_url: str | None = None) -> None:
|
|
444
447
|
self.url = f"{platform_url.rstrip('/')}{LEDGER_ROUTE}"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""The knob values a session opens with (a world state's bindings), as the World Host takes them."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .validation import fail
|
|
6
|
+
|
|
7
|
+
MAX_RUN_PARAMETERS = 8
|
|
8
|
+
MAX_RUN_PARAMETER_VALUE_LENGTH = 64
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def parse_run_parameters(raw: object) -> dict[str, str]:
|
|
12
|
+
if raw is None:
|
|
13
|
+
return {}
|
|
14
|
+
if not isinstance(raw, dict):
|
|
15
|
+
fail("host", "parameters is an object of parameter name to value")
|
|
16
|
+
if len(raw) > MAX_RUN_PARAMETERS:
|
|
17
|
+
fail("host", f"an open binds at most {MAX_RUN_PARAMETERS} parameters, not {len(raw)}")
|
|
18
|
+
for name, value in raw.items():
|
|
19
|
+
if not isinstance(name, str) or not name:
|
|
20
|
+
fail("host", "each parameter has a name")
|
|
21
|
+
if not isinstance(value, str):
|
|
22
|
+
fail("host", f"parameter '{name}' is bound to a string, not {type(value).__name__}")
|
|
23
|
+
if len(value) > MAX_RUN_PARAMETER_VALUE_LENGTH:
|
|
24
|
+
fail("host", f"parameter '{name}' is bound to at most {MAX_RUN_PARAMETER_VALUE_LENGTH} characters")
|
|
25
|
+
return dict(raw)
|