@withgateway/sdk 0.6.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +581 -0
- package/dist/agents/index.d.ts +581 -0
- package/dist/agents/index.js +1473 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1424 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +11611 -6845
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +10090 -5325
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +3177 -748
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2987 -590
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1986 -1746
- package/dist/worlds/index.d.ts +1986 -1746
- package/dist/worlds/index.js +1470 -290
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1465 -286
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +94 -12
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
- package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +2 -2
- package/skills/connector-schema-authoring/connector-worlds.md +3 -2
- package/skills/connector-schema-authoring/world-schema.md +36 -23
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +8 -6
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -7,8 +7,9 @@ The stdio protocol deliberately contains bundle bytes and mapping context only.
|
|
|
7
7
|
|
|
8
8
|
# file limit exception: the bridge's command dispatcher and its import/snapshot/
|
|
9
9
|
# check handlers share one request-validation vocabulary (_workdir, _under,
|
|
10
|
-
# _write_tree, fail); the read commands
|
|
11
|
-
# the next split is the import/snapshot
|
|
10
|
+
# _write_tree, fail); the read commands live in platform_query.py, the template
|
|
11
|
+
# commands in platform_templates.py, and the next split is the import/snapshot
|
|
12
|
+
# half. This file must not grow.
|
|
12
13
|
|
|
13
14
|
from __future__ import annotations
|
|
14
15
|
|
|
@@ -26,20 +27,25 @@ from typing import TYPE_CHECKING, Any
|
|
|
26
27
|
|
|
27
28
|
from .._materialize import materialize_template
|
|
28
29
|
from ..world import WorldTree
|
|
29
|
-
from . import
|
|
30
|
+
from . import (
|
|
31
|
+
ContractError,
|
|
32
|
+
WorldDatabase,
|
|
33
|
+
bundle_mode,
|
|
34
|
+
check_bundle,
|
|
35
|
+
platform_import,
|
|
36
|
+
platform_query,
|
|
37
|
+
platform_templates,
|
|
38
|
+
)
|
|
30
39
|
from . import snapshot as snapshots
|
|
31
|
-
from .cast import world_fields
|
|
32
40
|
from .compiler import CompilePlan, retarget_journals
|
|
33
41
|
from .handler_scan import HandlerScan
|
|
34
42
|
from .redaction import check_world_redaction
|
|
35
43
|
from .scaffold import request_files
|
|
36
|
-
from .validation import encoded, fail
|
|
44
|
+
from .validation import encoded, fail, tree_path
|
|
37
45
|
|
|
38
46
|
if TYPE_CHECKING:
|
|
39
47
|
from .compiler import Contract
|
|
40
|
-
from .connector import Spec
|
|
41
48
|
from .dependencies import FieldMap
|
|
42
|
-
from .redaction import RedactPolicy
|
|
43
49
|
|
|
44
50
|
MAX_BYTES = 64 * 1024 * 1024
|
|
45
51
|
MAX_FILES = 2000
|
|
@@ -48,6 +54,7 @@ COMMANDS = (
|
|
|
48
54
|
"validate-archive",
|
|
49
55
|
"init-template",
|
|
50
56
|
"init-template-files",
|
|
57
|
+
"init-world-files",
|
|
51
58
|
"init-openapi",
|
|
52
59
|
"template-summary",
|
|
53
60
|
"describe",
|
|
@@ -60,6 +67,9 @@ COMMANDS = (
|
|
|
60
67
|
"chunk-files",
|
|
61
68
|
"ingest",
|
|
62
69
|
"ingest-calls",
|
|
70
|
+
"map-rows",
|
|
71
|
+
"redact-rows",
|
|
72
|
+
"redact-policy",
|
|
63
73
|
"conform",
|
|
64
74
|
"reconcile",
|
|
65
75
|
"counts",
|
|
@@ -397,11 +407,9 @@ def _write_files(root: Path, entries: list) -> None:
|
|
|
397
407
|
if not isinstance(entry, dict) or set(entry) != {"path", "content"}:
|
|
398
408
|
fail("input", "Expected path and base64 content")
|
|
399
409
|
name = entry["path"]
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
if path.is_absolute() or any(p in ("", ".", "..") for p in name.split("/")) or name in seen:
|
|
404
|
-
fail("path", "Bundle paths must be unique relative file paths")
|
|
410
|
+
path = tree_path(name)
|
|
411
|
+
if name in seen:
|
|
412
|
+
fail("path", "Bundle paths must be unique")
|
|
405
413
|
seen.add(name)
|
|
406
414
|
try:
|
|
407
415
|
data = base64.b64decode(entry["content"], validate=True)
|
|
@@ -512,203 +520,6 @@ def _capture(request: dict) -> dict:
|
|
|
512
520
|
}
|
|
513
521
|
|
|
514
522
|
|
|
515
|
-
def _redact_summary(spec: "Spec | RedactPolicy") -> dict:
|
|
516
|
-
"""``slug`` is the salt ``_digest`` hashes under: the connector's slug, not the platform world slug."""
|
|
517
|
-
return {
|
|
518
|
-
"slug": spec.slug,
|
|
519
|
-
"hash": sorted(spec.hash_fields),
|
|
520
|
-
"drop": sorted(spec.drop_fields),
|
|
521
|
-
"preserve": sorted(spec.preserve_fields),
|
|
522
|
-
"round": dict(spec.round_fields),
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
def _annotate_redaction(entities: list[dict], spec: "Spec | RedactPolicy | None") -> None:
|
|
527
|
-
from .connector import redaction_of
|
|
528
|
-
|
|
529
|
-
for entity in entities:
|
|
530
|
-
for field in entity["fields"]:
|
|
531
|
-
field["redaction"] = redaction_of(spec, field["name"]) if spec else None
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
def _template_summary(files: dict[str, bytes]) -> dict:
|
|
535
|
-
"""Never runs the template's Python."""
|
|
536
|
-
from .connector import bind_spec, parse_spec
|
|
537
|
-
from .redaction import parse_env_policy
|
|
538
|
-
from .scaffold import template_digest
|
|
539
|
-
|
|
540
|
-
world_path = next((path for path in ("world.json", "schema/world.json") if path in files), None)
|
|
541
|
-
if world_path is None:
|
|
542
|
-
fail(
|
|
543
|
-
"template",
|
|
544
|
-
"No world.json (template) or schema/world.json (world) among the files",
|
|
545
|
-
)
|
|
546
|
-
try:
|
|
547
|
-
world = json.loads(files[world_path].decode("utf-8"))
|
|
548
|
-
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
549
|
-
fail("template", f"{world_path} is not valid JSON: {error}")
|
|
550
|
-
gateway = world.get("x-gateway") if isinstance(world, dict) else None
|
|
551
|
-
if not isinstance(gateway, dict):
|
|
552
|
-
fail("template", f"{world_path} is not a gateway-world document")
|
|
553
|
-
spec = bind_spec(parse_spec(files["connector.toml"].decode("utf-8")), world) if "connector.toml" in files else None
|
|
554
|
-
policy = spec
|
|
555
|
-
if policy is None and "gateway-env.toml" in files:
|
|
556
|
-
policy = parse_env_policy(files["gateway-env.toml"].decode("utf-8"))
|
|
557
|
-
entities = []
|
|
558
|
-
fields_by_entity = world_fields(world)
|
|
559
|
-
for name, entity in sorted((gateway.get("entities") or {}).items()):
|
|
560
|
-
entities.append(
|
|
561
|
-
{
|
|
562
|
-
"name": name,
|
|
563
|
-
"primaryKey": list(entity.get("primary_key") or []) if isinstance(entity, dict) else [],
|
|
564
|
-
"unique": list(entity.get("unique") or []) if isinstance(entity, dict) else [],
|
|
565
|
-
"relationships": list(entity.get("relationships") or []) if isinstance(entity, dict) else [],
|
|
566
|
-
"fields": [field.describe() for field in fields_by_entity.get(name, [])],
|
|
567
|
-
}
|
|
568
|
-
)
|
|
569
|
-
_annotate_redaction(entities, policy)
|
|
570
|
-
connection = None
|
|
571
|
-
if spec is not None:
|
|
572
|
-
connection = {
|
|
573
|
-
"slug": spec.slug,
|
|
574
|
-
"baseUrl": spec.base_url,
|
|
575
|
-
"auth": spec.scheme,
|
|
576
|
-
"redact": _redact_summary(spec),
|
|
577
|
-
"operations": [
|
|
578
|
-
{
|
|
579
|
-
"name": op.name,
|
|
580
|
-
"method": op.method,
|
|
581
|
-
"path": op.path,
|
|
582
|
-
"params": list(op.params),
|
|
583
|
-
"entity": op.entity,
|
|
584
|
-
"results": op.results,
|
|
585
|
-
}
|
|
586
|
-
for op in spec.operations.values()
|
|
587
|
-
],
|
|
588
|
-
}
|
|
589
|
-
package = gateway.get("package") or {}
|
|
590
|
-
return {
|
|
591
|
-
"status": "ok",
|
|
592
|
-
"summary": {
|
|
593
|
-
"packageVersion": package.get("version"),
|
|
594
|
-
"domainVersion": gateway.get("domain_version"),
|
|
595
|
-
"domainId": world.get("$id"),
|
|
596
|
-
"digest": template_digest(files),
|
|
597
|
-
"entities": entities,
|
|
598
|
-
"connection": connection,
|
|
599
|
-
"redact": _redact_summary(policy) if policy is not None else None,
|
|
600
|
-
"tools": sorted((gateway.get("tools") or {}).keys()),
|
|
601
|
-
"files": sorted(files),
|
|
602
|
-
},
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
COMPILED_MARKERS = ("schema/world.json", "tool-schemas.json")
|
|
607
|
-
DESCRIBE_PATHS = (
|
|
608
|
-
"schema/world.json",
|
|
609
|
-
"tool-schemas.json",
|
|
610
|
-
"connector.toml",
|
|
611
|
-
"gateway-env.toml",
|
|
612
|
-
"api/handlers.py",
|
|
613
|
-
)
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
def _describe(request: dict) -> dict:
|
|
617
|
-
"""Never runs the connector's Python."""
|
|
618
|
-
from .connector import load_spec, parse_spec
|
|
619
|
-
from .redaction import env_policy
|
|
620
|
-
from .scaffold import (
|
|
621
|
-
TEMPLATES,
|
|
622
|
-
init_bundle,
|
|
623
|
-
init_bundle_from_files,
|
|
624
|
-
template_digest,
|
|
625
|
-
)
|
|
626
|
-
from .ui import ui_kind
|
|
627
|
-
|
|
628
|
-
connector = request.get("connector")
|
|
629
|
-
with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
|
|
630
|
-
root = Path(directory) / "bundle"
|
|
631
|
-
if connector is not None:
|
|
632
|
-
if not isinstance(connector, str):
|
|
633
|
-
fail("input", "Expected an installed connector template slug")
|
|
634
|
-
init_bundle(root, connector=connector)
|
|
635
|
-
template_dir = TEMPLATES / connector
|
|
636
|
-
authored = {
|
|
637
|
-
str(path.relative_to(template_dir)): path.read_bytes()
|
|
638
|
-
for path in sorted(template_dir.rglob("*"))
|
|
639
|
-
if path.is_file() and "__pycache__" not in path.parts
|
|
640
|
-
}
|
|
641
|
-
else:
|
|
642
|
-
authored = request_files(request)
|
|
643
|
-
if all(marker in authored for marker in COMPILED_MARKERS):
|
|
644
|
-
root.mkdir()
|
|
645
|
-
for path, data in authored.items():
|
|
646
|
-
if path in DESCRIBE_PATHS:
|
|
647
|
-
target = root / path
|
|
648
|
-
target.parent.mkdir(parents=True, exist_ok=True)
|
|
649
|
-
target.write_bytes(data)
|
|
650
|
-
else:
|
|
651
|
-
slug = request.get("slug") if isinstance(request.get("slug"), str) else None
|
|
652
|
-
if slug is None and "connector.toml" in authored:
|
|
653
|
-
slug = parse_spec(authored["connector.toml"].decode("utf-8")).slug
|
|
654
|
-
init_bundle_from_files(root, authored, connector=slug or "custom")
|
|
655
|
-
summary = _template_summary({"world.json": (root / "schema/world.json").read_bytes()})["summary"]
|
|
656
|
-
summary["digest"] = template_digest(authored)
|
|
657
|
-
summary["files"] = sorted(authored)
|
|
658
|
-
summary["ui"] = ui_kind(root)
|
|
659
|
-
tool_schemas = {}
|
|
660
|
-
if (root / "tool-schemas.json").is_file():
|
|
661
|
-
tool_schemas = json.loads((root / "tool-schemas.json").read_text(encoding="utf-8")).get("tools") or {}
|
|
662
|
-
summary["tools"] = [
|
|
663
|
-
{
|
|
664
|
-
"name": name,
|
|
665
|
-
"description": str(entry.get("description") or ""),
|
|
666
|
-
"inputSchema": entry.get("input_schema"),
|
|
667
|
-
"outputSchema": entry.get("output_schema"),
|
|
668
|
-
}
|
|
669
|
-
for name, entry in sorted(tool_schemas.items())
|
|
670
|
-
]
|
|
671
|
-
summary["connection"] = None
|
|
672
|
-
summary["openapi"] = None
|
|
673
|
-
if (root / "connector.toml").is_file():
|
|
674
|
-
from .api import load_api_settings, openapi_document
|
|
675
|
-
|
|
676
|
-
spec = load_spec(root)
|
|
677
|
-
try:
|
|
678
|
-
settings = load_api_settings(root, spec)
|
|
679
|
-
except ContractError:
|
|
680
|
-
# A template without its handlers still has a shape worth showing.
|
|
681
|
-
settings = None
|
|
682
|
-
_annotate_redaction(summary["entities"], spec)
|
|
683
|
-
summary["connection"] = {
|
|
684
|
-
"slug": spec.slug,
|
|
685
|
-
"baseUrl": spec.base_url,
|
|
686
|
-
"auth": spec.scheme,
|
|
687
|
-
"redact": _redact_summary(spec),
|
|
688
|
-
"operations": [
|
|
689
|
-
{
|
|
690
|
-
"name": op.name,
|
|
691
|
-
"method": op.method,
|
|
692
|
-
"path": op.path,
|
|
693
|
-
"params": list(op.params),
|
|
694
|
-
"entity": op.entity,
|
|
695
|
-
"results": op.results,
|
|
696
|
-
"action": settings.for_operation(op.name).action if settings else None,
|
|
697
|
-
"status": settings.for_operation(op.name).status if settings else None,
|
|
698
|
-
}
|
|
699
|
-
for op in spec.operations.values()
|
|
700
|
-
],
|
|
701
|
-
}
|
|
702
|
-
summary["openapi"] = openapi_document(spec, summary.get("domainId") or spec.slug, settings)
|
|
703
|
-
summary["redact"] = summary["connection"]["redact"]
|
|
704
|
-
else:
|
|
705
|
-
policy = env_policy(root / "gateway-env.toml")
|
|
706
|
-
_annotate_redaction(summary["entities"], policy)
|
|
707
|
-
summary["redact"] = _redact_summary(policy) if policy else None
|
|
708
|
-
summary["dependencies"] = describe_dependencies(root) if (root / "gateway-env.toml").is_file() else []
|
|
709
|
-
return {"status": "ok", "description": summary}
|
|
710
|
-
|
|
711
|
-
|
|
712
523
|
def _ingest_calls(request: dict) -> dict:
|
|
713
524
|
"""Transforms never run here: they are the caller's code and run where the caller is."""
|
|
714
525
|
from .data import KIND_SCHEMA, MODES
|
|
@@ -752,6 +563,52 @@ def _ingest_calls(request: dict) -> dict:
|
|
|
752
563
|
return answer
|
|
753
564
|
|
|
754
565
|
|
|
566
|
+
def _map_rows(request: dict) -> dict:
|
|
567
|
+
"""Rows a declared ``ingest.toml`` maps the records to, for the Data Factory's upload, which lands them itself."""
|
|
568
|
+
from .ingest import map_records
|
|
569
|
+
|
|
570
|
+
records = request.get("records")
|
|
571
|
+
if not isinstance(records, list):
|
|
572
|
+
fail("input", "Expected records as a list of objects")
|
|
573
|
+
mapping = request.get("map")
|
|
574
|
+
if not isinstance(mapping, str):
|
|
575
|
+
fail("input", "map is an ingest.toml text")
|
|
576
|
+
with tempfile.TemporaryDirectory(prefix="gateway-map-rows-") as directory:
|
|
577
|
+
root = Path(directory) / "bundle"
|
|
578
|
+
root.mkdir()
|
|
579
|
+
_write_files(root, request.get("files"))
|
|
580
|
+
projected, report = map_records(world_root(root), records, mapping)
|
|
581
|
+
return {"status": "ok", "projected": projected, **report}
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _redact_rows(request: dict) -> dict:
|
|
585
|
+
"""Rows under the world's ``[redact]`` policy, for the Data Factory's upload, which plans and checks them in the form the world stores."""
|
|
586
|
+
from .redaction import parse_mode, redact_rows, require_policy
|
|
587
|
+
|
|
588
|
+
rows = request.get("rows")
|
|
589
|
+
if not isinstance(rows, dict) or not all(isinstance(listed, list) for listed in rows.values()):
|
|
590
|
+
fail("input", "Expected rows as {entity: [rows]}")
|
|
591
|
+
mode = parse_mode(request.get("redact"))
|
|
592
|
+
with tempfile.TemporaryDirectory(prefix="gateway-redact-rows-") as directory:
|
|
593
|
+
root = Path(directory) / "bundle"
|
|
594
|
+
root.mkdir()
|
|
595
|
+
_write_files(root, request.get("files"))
|
|
596
|
+
redacted = redact_rows(require_policy(world_root(root), mode), rows, mode)
|
|
597
|
+
return {"status": "ok", "rows": redacted}
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _redact_policy(request: dict) -> dict:
|
|
601
|
+
"""The ``[redact]`` policy this world's files declare, read exactly as ``redact-rows`` reads it (``policy_for``), or null when there is none."""
|
|
602
|
+
from .redaction import policy_for, policy_view
|
|
603
|
+
|
|
604
|
+
with tempfile.TemporaryDirectory(prefix="gateway-redact-policy-") as directory:
|
|
605
|
+
root = Path(directory) / "bundle"
|
|
606
|
+
root.mkdir()
|
|
607
|
+
_write_files(root, request.get("files"))
|
|
608
|
+
policy = policy_view(policy_for(world_root(root)))
|
|
609
|
+
return {"status": "ok", "policy": policy}
|
|
610
|
+
|
|
611
|
+
|
|
755
612
|
def _workdir(request: dict) -> Path:
|
|
756
613
|
workdir = request.get("workdir")
|
|
757
614
|
if not isinstance(workdir, str) or not workdir or not Path(workdir).is_absolute() or not Path(workdir).is_dir():
|
|
@@ -1013,8 +870,8 @@ def _conform(request: dict) -> dict:
|
|
|
1013
870
|
|
|
1014
871
|
|
|
1015
872
|
def _run_tests(request: dict) -> dict:
|
|
873
|
+
from .measured_tests import MeasuredTestRun
|
|
1016
874
|
from .snapshot import fetch as fetch_snapshot
|
|
1017
|
-
from .world_tests import run
|
|
1018
875
|
|
|
1019
876
|
scenario = request.get("scenario")
|
|
1020
877
|
if scenario is not None and not isinstance(scenario, str):
|
|
@@ -1028,8 +885,10 @@ def _run_tests(request: dict) -> dict:
|
|
|
1028
885
|
root = _world_for(request, Path(directory))
|
|
1029
886
|
if snapshot is not None:
|
|
1030
887
|
fetch_snapshot(root, check_bundle(root), snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
1031
|
-
|
|
1032
|
-
report
|
|
888
|
+
measured = MeasuredTestRun.of(root, scenario)
|
|
889
|
+
report = {**measured.report, "world": root.name}
|
|
890
|
+
if measured.peak_mb is not None:
|
|
891
|
+
report["memory"] = {"peakMb": measured.peak_mb}
|
|
1033
892
|
return {"status": "ok", "report": report}
|
|
1034
893
|
|
|
1035
894
|
|
|
@@ -1159,31 +1018,6 @@ def _dry_run(request: dict) -> dict:
|
|
|
1159
1018
|
return answer
|
|
1160
1019
|
|
|
1161
1020
|
|
|
1162
|
-
def _templates(request: dict) -> dict:
|
|
1163
|
-
"""Never runs a template's Python."""
|
|
1164
|
-
from .scaffold import TEMPLATES, list_templates
|
|
1165
|
-
from .ui import ui_kind
|
|
1166
|
-
|
|
1167
|
-
templates = []
|
|
1168
|
-
for slug in list_templates():
|
|
1169
|
-
world = json.loads((TEMPLATES / slug / "world.json").read_text(encoding="utf-8"))
|
|
1170
|
-
gateway = world.get("x-gateway") if isinstance(world, dict) else None
|
|
1171
|
-
if not isinstance(gateway, dict):
|
|
1172
|
-
fail("template", f"{slug}/world.json is not a gateway-world document")
|
|
1173
|
-
templates.append(
|
|
1174
|
-
{
|
|
1175
|
-
"slug": slug,
|
|
1176
|
-
"name": world.get("title") if isinstance(world.get("title"), str) else None,
|
|
1177
|
-
"description": world.get("description") if isinstance(world.get("description"), str) else None,
|
|
1178
|
-
"entities": sorted(gateway.get("entities") or {}),
|
|
1179
|
-
"tools": sorted(gateway.get("tools") or {}),
|
|
1180
|
-
"hasConnection": (TEMPLATES / slug / "connector.toml").is_file(),
|
|
1181
|
-
"ui": ui_kind(TEMPLATES / slug),
|
|
1182
|
-
}
|
|
1183
|
-
)
|
|
1184
|
-
return {"status": "ok", "templates": templates}
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
1021
|
def _answering_commands() -> dict:
|
|
1188
1022
|
return {
|
|
1189
1023
|
"validate-rows": _validate_rows,
|
|
@@ -1195,6 +1029,9 @@ def _answering_commands() -> dict:
|
|
|
1195
1029
|
"chunk-files": lambda request: platform_import.chunk_files(request, _world_for),
|
|
1196
1030
|
"ingest": _ingest,
|
|
1197
1031
|
"ingest-calls": _ingest_calls,
|
|
1032
|
+
"map-rows": _map_rows,
|
|
1033
|
+
"redact-rows": _redact_rows,
|
|
1034
|
+
"redact-policy": _redact_policy,
|
|
1198
1035
|
"conform": _conform,
|
|
1199
1036
|
"reconcile": _reconcile,
|
|
1200
1037
|
"run-tests": _run_tests,
|
|
@@ -1204,10 +1041,10 @@ def _answering_commands() -> dict:
|
|
|
1204
1041
|
"query": lambda request: platform_query.query(request, _world_for),
|
|
1205
1042
|
"sql": lambda request: platform_query.sql(request, _world_for),
|
|
1206
1043
|
"verify-legacy": lambda request: platform_query.verify_legacy(request, _world_for),
|
|
1207
|
-
"templates":
|
|
1044
|
+
"templates": platform_templates.templates,
|
|
1208
1045
|
"api-bundle": _api_bundle,
|
|
1209
|
-
"template-summary": lambda request:
|
|
1210
|
-
"describe":
|
|
1046
|
+
"template-summary": lambda request: platform_templates.template_summary(request_files(request)),
|
|
1047
|
+
"describe": platform_templates.describe,
|
|
1211
1048
|
}
|
|
1212
1049
|
|
|
1213
1050
|
|
|
@@ -1240,28 +1077,9 @@ def execute(request: dict) -> dict:
|
|
|
1240
1077
|
from .api import write_base_seed
|
|
1241
1078
|
|
|
1242
1079
|
write_base_seed(world, root)
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
connector = request.get("connector")
|
|
1247
|
-
name = request.get("name")
|
|
1248
|
-
if not isinstance(connector, str) or (name is not None and not isinstance(name, str)):
|
|
1249
|
-
fail("input", "Expected a connector template slug and an optional name")
|
|
1250
|
-
init_bundle(
|
|
1251
|
-
root,
|
|
1252
|
-
connector=connector,
|
|
1253
|
-
name=name,
|
|
1254
|
-
with_example_tool=request.get("with_example_tool") is True,
|
|
1255
|
-
)
|
|
1256
|
-
if request["command"] == "init-template-files":
|
|
1257
|
-
# The template arrives as files, never as a path the request chose.
|
|
1258
|
-
from .scaffold import init_bundle_from_files
|
|
1259
|
-
|
|
1260
|
-
connector = request.get("connector")
|
|
1261
|
-
name = request.get("name")
|
|
1262
|
-
if not isinstance(connector, str) or (name is not None and not isinstance(name, str)):
|
|
1263
|
-
fail("input", "Expected the connector slug and an optional name")
|
|
1264
|
-
init_bundle_from_files(root, request_files(request), connector=connector, name=name)
|
|
1080
|
+
writer = platform_templates.WRITERS.get(request["command"])
|
|
1081
|
+
if writer is not None:
|
|
1082
|
+
writer(request, root)
|
|
1265
1083
|
if request["command"] == "init-openapi":
|
|
1266
1084
|
from .openapi_import import OpenApiImport
|
|
1267
1085
|
|
|
@@ -1269,14 +1087,6 @@ def execute(request: dict) -> dict:
|
|
|
1269
1087
|
if not planned["written"]:
|
|
1270
1088
|
return planned
|
|
1271
1089
|
extra.update(planned)
|
|
1272
|
-
if request["command"] == "build-slack":
|
|
1273
|
-
from .slack import build_slack_bundle
|
|
1274
|
-
|
|
1275
|
-
build_slack_bundle(root, evidence=request.get("evidence"), name=request.get("name"))
|
|
1276
|
-
if request["command"] == "refresh-slack":
|
|
1277
|
-
from .slack import refresh_slack_bundle
|
|
1278
|
-
|
|
1279
|
-
refresh_slack_bundle(root, request.get("evidence"), batch_id=request.get("batch_id"))
|
|
1280
1090
|
report = {
|
|
1281
1091
|
**validate_bundle(world_root(root), execute=_flag(request, "execute", False)),
|
|
1282
1092
|
**extra,
|
|
@@ -20,6 +20,7 @@ from typing import Any, Callable
|
|
|
20
20
|
from ..legacy_sqlite import LegacyVerifiers
|
|
21
21
|
from .compiler import bundle_mode
|
|
22
22
|
from .data import QUERY_DEFAULT_LIMIT, entity_counts, entity_query, entity_query_across, entity_rows, world_kind
|
|
23
|
+
from .process_memory import SIZE, ProcessStatus
|
|
23
24
|
from .validation import ContractError, fail
|
|
24
25
|
from .world_sql import WorldSql
|
|
25
26
|
|
|
@@ -127,10 +128,10 @@ def cap_memory_for(request: Any, status: Path = Path("/proc/self/status"), limit
|
|
|
127
128
|
return
|
|
128
129
|
if limits is None:
|
|
129
130
|
import resource as limits
|
|
130
|
-
|
|
131
|
-
if
|
|
131
|
+
size = ProcessStatus.read(status).bytes(SIZE)
|
|
132
|
+
if size is None:
|
|
132
133
|
fail("runtime", "This process cannot read its own memory size, so it refuses to run caller SQL")
|
|
133
|
-
cap =
|
|
134
|
+
cap = size + SQL_MEMORY_BYTES
|
|
134
135
|
soft, hard = limits.getrlimit(limits.RLIMIT_AS)
|
|
135
136
|
hard = _lower(cap, hard, limits.RLIM_INFINITY)
|
|
136
137
|
limits.setrlimit(limits.RLIMIT_AS, (_lower(hard, soft, limits.RLIM_INFINITY), hard))
|