@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
|
@@ -5,11 +5,14 @@ inside the hosted runner or an isolated workbench, never the publication server.
|
|
|
5
5
|
The stdio protocol deliberately contains bundle bytes and mapping context only.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
# file limit exception: the bridge's command dispatcher and its import/snapshot/
|
|
9
|
+
# check handlers share one request-validation vocabulary (_workdir, _under,
|
|
10
|
+
# _write_tree, fail); the read commands already live in platform_query.py and
|
|
11
|
+
# the next split is the import/snapshot half. This file must not grow.
|
|
12
|
+
|
|
8
13
|
from __future__ import annotations
|
|
9
14
|
|
|
10
|
-
import ast
|
|
11
15
|
import base64
|
|
12
|
-
import binascii
|
|
13
16
|
import gzip
|
|
14
17
|
import io
|
|
15
18
|
import json
|
|
@@ -18,15 +21,19 @@ import shutil
|
|
|
18
21
|
import sys
|
|
19
22
|
import tarfile
|
|
20
23
|
import tempfile
|
|
24
|
+
from dataclasses import asdict
|
|
21
25
|
from pathlib import Path, PurePosixPath
|
|
22
26
|
from typing import TYPE_CHECKING, Any
|
|
23
27
|
|
|
24
28
|
from .._materialize import materialize_template
|
|
25
29
|
from ..world import WorldTree
|
|
26
|
-
from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle
|
|
30
|
+
from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle, platform_import, platform_query
|
|
27
31
|
from . import snapshot as snapshots
|
|
32
|
+
from .cast import world_fields
|
|
28
33
|
from .compiler import retarget_empty_journals
|
|
34
|
+
from .handler_scan import HandlerScan
|
|
29
35
|
from .redaction import check_world_redaction
|
|
36
|
+
from .scaffold import request_files
|
|
30
37
|
from .validation import (
|
|
31
38
|
digest,
|
|
32
39
|
encoded,
|
|
@@ -50,6 +57,7 @@ COMMANDS = (
|
|
|
50
57
|
"validate-archive",
|
|
51
58
|
"init-template",
|
|
52
59
|
"init-template-files",
|
|
60
|
+
"init-openapi",
|
|
53
61
|
"template-summary",
|
|
54
62
|
"describe",
|
|
55
63
|
"compile",
|
|
@@ -58,6 +66,8 @@ COMMANDS = (
|
|
|
58
66
|
"refresh-slack",
|
|
59
67
|
"capture",
|
|
60
68
|
"import-data",
|
|
69
|
+
"import-files",
|
|
70
|
+
"chunk-files",
|
|
61
71
|
"ingest",
|
|
62
72
|
"ingest-calls",
|
|
63
73
|
"conform",
|
|
@@ -238,62 +248,7 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
|
|
|
238
248
|
fail("mode", "Expected a schema world: gateway-env.toml with [schema] beside schema/world.json")
|
|
239
249
|
contract = check_bundle(bundle)
|
|
240
250
|
declared = set(contract.tools)
|
|
241
|
-
|
|
242
|
-
for path in sorted((bundle / "tools").glob("*.py")):
|
|
243
|
-
if path.is_symlink():
|
|
244
|
-
fail("path", "Tool handlers must be regular files")
|
|
245
|
-
try:
|
|
246
|
-
tree = ast.parse(path.read_text(encoding="utf-8"), filename=path.name)
|
|
247
|
-
except (SyntaxError, UnicodeError):
|
|
248
|
-
fail("handler", f"Invalid Python handler syntax: {path.name}")
|
|
249
|
-
for node in tree.body:
|
|
250
|
-
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
251
|
-
continue
|
|
252
|
-
names = []
|
|
253
|
-
for decorator in node.decorator_list:
|
|
254
|
-
target = decorator.func if isinstance(decorator, ast.Call) else decorator
|
|
255
|
-
is_tool = (isinstance(target, ast.Name) and target.id == "tool") or (
|
|
256
|
-
isinstance(target, ast.Attribute) and target.attr == "tool"
|
|
257
|
-
)
|
|
258
|
-
if not is_tool:
|
|
259
|
-
continue
|
|
260
|
-
name = node.name
|
|
261
|
-
if isinstance(decorator, ast.Call):
|
|
262
|
-
for keyword in decorator.keywords:
|
|
263
|
-
if keyword.arg == "name":
|
|
264
|
-
if not isinstance(keyword.value, ast.Constant) or not isinstance(keyword.value.value, str):
|
|
265
|
-
fail("handler", "Publication requires literal tool names")
|
|
266
|
-
name = keyword.value.value
|
|
267
|
-
names.append(name)
|
|
268
|
-
if not names and node.name in declared:
|
|
269
|
-
names.append(node.name)
|
|
270
|
-
if not names and node.name == "run" and path.stem in declared:
|
|
271
|
-
names.append(path.stem)
|
|
272
|
-
for name in names:
|
|
273
|
-
if name in found:
|
|
274
|
-
fail("handler", f"Duplicate handler for {name}")
|
|
275
|
-
found[name] = path.name
|
|
276
|
-
if set(found) != declared:
|
|
277
|
-
undeclared = sorted(set(found) - declared)
|
|
278
|
-
unhandled = sorted(declared - set(found))
|
|
279
|
-
parts = []
|
|
280
|
-
if unhandled:
|
|
281
|
-
parts.append(
|
|
282
|
-
"declared in schema/world.json (tools) without a handler under tools/: "
|
|
283
|
-
+ ", ".join(unhandled)
|
|
284
|
-
+ " — add `@tool` def <name>(...) to a tools/*.py, or drop the declaration"
|
|
285
|
-
)
|
|
286
|
-
if undeclared:
|
|
287
|
-
parts.append(
|
|
288
|
-
"handled under tools/ without a declaration: "
|
|
289
|
-
+ ", ".join(f"{name} ({found[name]})" for name in undeclared)
|
|
290
|
-
+ " — declare each in schema/world.json, or remove the handler"
|
|
291
|
-
)
|
|
292
|
-
fail(
|
|
293
|
-
"handler",
|
|
294
|
-
"Declared tools and Python handlers must match exactly; " + "; ".join(parts),
|
|
295
|
-
"/tools",
|
|
296
|
-
)
|
|
251
|
+
HandlerScan(bundle, declared).run()
|
|
297
252
|
if (bundle / "connector.toml").is_file():
|
|
298
253
|
from .api import load_api_settings
|
|
299
254
|
from .connector import parse_spec
|
|
@@ -393,12 +348,10 @@ def _check_verifiers(bundle: Path, store: SQLiteStore, unattached: dict[str, str
|
|
|
393
348
|
"""Every ``verifiers/*.sql`` run over the template through the store's
|
|
394
349
|
read-only authorizer, the way a session grades. A verifier that names a
|
|
395
350
|
table or column the contract lacks is refused here — at check — not at
|
|
396
|
-
the first grade. ``scored`` reports the score over the initial state
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
platform opens the link."""
|
|
400
|
-
from .._world import split_sql_statements
|
|
401
|
-
|
|
351
|
+
the first grade. ``scored`` reports the score over the initial state by
|
|
352
|
+
the one :class:`SqlVerifier` rule a grade applies. A verifier that reads
|
|
353
|
+
a dependency the check could not attach (a pin without a local path) is
|
|
354
|
+
reported, not refused: it runs where the platform opens the link."""
|
|
402
355
|
verifiers_dir = bundle / "verifiers"
|
|
403
356
|
if not verifiers_dir.is_dir():
|
|
404
357
|
return {}
|
|
@@ -406,42 +359,48 @@ def _check_verifiers(bundle: Path, store: SQLiteStore, unattached: dict[str, str
|
|
|
406
359
|
for path in sorted(verifiers_dir.glob("*.sql")):
|
|
407
360
|
if path.is_symlink():
|
|
408
361
|
fail("path", "Verifiers must be regular files", f"/verifiers/{path.name}")
|
|
409
|
-
|
|
410
|
-
needs: str | None = None
|
|
411
|
-
for statement in split_sql_statements(path.read_text(encoding="utf-8")):
|
|
412
|
-
try:
|
|
413
|
-
rows = store.query(statement, private=True)
|
|
414
|
-
except ContractError as error:
|
|
415
|
-
reason = error.__cause__ if error.__cause__ is not None else error.detail["message"]
|
|
416
|
-
alias = _missing_dependency_alias(str(reason), unattached)
|
|
417
|
-
if alias is not None:
|
|
418
|
-
needs = alias
|
|
419
|
-
break
|
|
420
|
-
hint = ""
|
|
421
|
-
if "no such table" in str(reason):
|
|
422
|
-
tables = sorted(store.contract.entities)
|
|
423
|
-
tables += [f"{a}.{t}" for a, ts in sorted(store.attached.items()) for t in sorted(ts)]
|
|
424
|
-
hint = f"; the world's entities are {', '.join(tables) or '(none)'}"
|
|
425
|
-
fail(
|
|
426
|
-
"verifier",
|
|
427
|
-
f"verifiers/{path.name} does not run against this schema: {reason}{hint}",
|
|
428
|
-
f"/verifiers/{path.name}",
|
|
429
|
-
)
|
|
430
|
-
if rows:
|
|
431
|
-
score = rows[0][0]
|
|
432
|
-
if needs is not None:
|
|
433
|
-
out[path.stem] = {
|
|
434
|
-
"status": "needs_dependency",
|
|
435
|
-
"dependency": needs,
|
|
436
|
-
"reason": f"reads {needs}.<table>; {unattached[needs]}",
|
|
437
|
-
}
|
|
438
|
-
elif scored:
|
|
439
|
-
out[path.stem] = {"status": "ok", "initial_score": score}
|
|
440
|
-
else:
|
|
441
|
-
out[path.stem] = {"status": "ok"}
|
|
362
|
+
out[path.stem] = _check_verifier(path, store, unattached, scored=scored)
|
|
442
363
|
return out
|
|
443
364
|
|
|
444
365
|
|
|
366
|
+
def _check_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str], *, scored: bool) -> dict[str, Any]:
|
|
367
|
+
from .._world import SqlVerifier, VerifierError
|
|
368
|
+
|
|
369
|
+
try:
|
|
370
|
+
score, _sql = SqlVerifier.read(path).score(lambda statement: store.query(statement, private=True))
|
|
371
|
+
except ContractError as error:
|
|
372
|
+
alias = _refuse_verifier(path, store, unattached, error)
|
|
373
|
+
return {
|
|
374
|
+
"status": "needs_dependency",
|
|
375
|
+
"dependency": alias,
|
|
376
|
+
"reason": f"reads {alias}.<table>; {unattached[alias]}",
|
|
377
|
+
}
|
|
378
|
+
except VerifierError as error:
|
|
379
|
+
fail("verifier", str(error), f"/verifiers/{path.name}")
|
|
380
|
+
if not scored:
|
|
381
|
+
return {"status": "ok"}
|
|
382
|
+
return {"status": "ok", "initial_score": score}
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _refuse_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str], error: ContractError) -> str:
|
|
386
|
+
"""The unattached dependency alias the failure names — the one refusal
|
|
387
|
+
check reports rather than raises; every other failure is refused here."""
|
|
388
|
+
reason = error.__cause__ if error.__cause__ is not None else error.detail["message"]
|
|
389
|
+
alias = _missing_dependency_alias(str(reason), unattached)
|
|
390
|
+
if alias is not None:
|
|
391
|
+
return alias
|
|
392
|
+
hint = ""
|
|
393
|
+
if "no such table" in str(reason):
|
|
394
|
+
tables = sorted(store.contract.entities)
|
|
395
|
+
tables += [f"{a}.{t}" for a, ts in sorted(store.attached.items()) for t in sorted(ts)]
|
|
396
|
+
hint = f"; the world's entities are {', '.join(tables) or '(none)'}"
|
|
397
|
+
fail(
|
|
398
|
+
"verifier",
|
|
399
|
+
f"verifiers/{path.name} does not run against this schema: {reason}{hint}",
|
|
400
|
+
f"/verifiers/{path.name}",
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
|
|
445
404
|
def _missing_dependency_alias(reason: str, unattached: dict[str, str]) -> str | None:
|
|
446
405
|
"""The alias of an unattached dependency a ``no such table: alias.table``
|
|
447
406
|
failure names, else ``None``."""
|
|
@@ -698,25 +657,6 @@ def _capture(request: dict) -> dict:
|
|
|
698
657
|
}
|
|
699
658
|
|
|
700
659
|
|
|
701
|
-
def _template_files(request: dict) -> dict[str, bytes]:
|
|
702
|
-
listed = request.get("files")
|
|
703
|
-
if not isinstance(listed, list) or not listed:
|
|
704
|
-
fail("input", "Expected the template's files as [{path, content}]")
|
|
705
|
-
files: dict[str, bytes] = {}
|
|
706
|
-
for entry in listed:
|
|
707
|
-
if (
|
|
708
|
-
not isinstance(entry, dict)
|
|
709
|
-
or not isinstance(entry.get("path"), str)
|
|
710
|
-
or not isinstance(entry.get("content"), str)
|
|
711
|
-
):
|
|
712
|
-
fail("input", "Each template file is {path, content} with base64 content")
|
|
713
|
-
try:
|
|
714
|
-
files[entry["path"]] = base64.b64decode(entry["content"], validate=True)
|
|
715
|
-
except (ValueError, binascii.Error):
|
|
716
|
-
fail("input", "Template file content must be base64")
|
|
717
|
-
return files
|
|
718
|
-
|
|
719
|
-
|
|
720
660
|
def _redact_summary(spec: "Spec | RedactPolicy") -> dict:
|
|
721
661
|
"""The ``[redact]`` table as describe reports it, so a caller can compute a
|
|
722
662
|
stored form from these lists without running Python. ``slug`` is the SALT —
|
|
@@ -748,7 +688,7 @@ def _template_summary(files: dict[str, bytes]) -> dict:
|
|
|
748
688
|
"""What a workspace connector is, read from its files: the vendor connection
|
|
749
689
|
(connector.toml) and the data model (world.json), plus the digest the platform
|
|
750
690
|
pins worlds to. Never runs the template's Python."""
|
|
751
|
-
from .connector import parse_spec
|
|
691
|
+
from .connector import bind_spec, parse_spec
|
|
752
692
|
from .redaction import parse_env_policy
|
|
753
693
|
from .scaffold import template_digest
|
|
754
694
|
|
|
@@ -766,40 +706,20 @@ def _template_summary(files: dict[str, bytes]) -> dict:
|
|
|
766
706
|
gateway = world.get("x-gateway") if isinstance(world, dict) else None
|
|
767
707
|
if not isinstance(gateway, dict):
|
|
768
708
|
fail("template", f"{world_path} is not a gateway-world document")
|
|
769
|
-
spec = parse_spec(files["connector.toml"].decode("utf-8")) if "connector.toml" in files else None
|
|
709
|
+
spec = bind_spec(parse_spec(files["connector.toml"].decode("utf-8")), world) if "connector.toml" in files else None
|
|
770
710
|
policy = spec
|
|
771
711
|
if policy is None and "gateway-env.toml" in files:
|
|
772
712
|
policy = parse_env_policy(files["gateway-env.toml"].decode("utf-8"))
|
|
773
713
|
entities = []
|
|
714
|
+
fields_by_entity = world_fields(world)
|
|
774
715
|
for name, entity in sorted((gateway.get("entities") or {}).items()):
|
|
775
|
-
fields: list = []
|
|
776
|
-
schema = entity.get("schema") if isinstance(entity, dict) else None
|
|
777
|
-
ref = schema.get("$ref") if isinstance(schema, dict) else None
|
|
778
|
-
definition = None
|
|
779
|
-
if isinstance(ref, str) and ref.startswith("#/$defs/"):
|
|
780
|
-
definition = (world.get("$defs") or {}).get(ref[len("#/$defs/") :])
|
|
781
|
-
elif isinstance(schema, dict) and isinstance(schema.get("properties"), dict):
|
|
782
|
-
# An inline payload (`schema entity add --schema '{…}'`) declares its
|
|
783
|
-
# fields right here; it is as much a field list as a $defs entry.
|
|
784
|
-
definition = schema
|
|
785
|
-
properties = (definition or {}).get("properties") or {} if isinstance(definition, dict) else {}
|
|
786
|
-
required = set((definition or {}).get("required") or []) if isinstance(definition, dict) else set()
|
|
787
|
-
for field_name, field_schema in properties.items():
|
|
788
|
-
kind = field_schema.get("type") if isinstance(field_schema, dict) else None
|
|
789
|
-
if isinstance(kind, list):
|
|
790
|
-
# A nullable field is `["string", "null"]` in JSON Schema; the page
|
|
791
|
-
# shows it as one readable type, not the list's concatenation.
|
|
792
|
-
kind = " | ".join(str(part) for part in kind)
|
|
793
|
-
if kind is None and isinstance(field_schema, dict) and isinstance(field_schema.get("$ref"), str):
|
|
794
|
-
kind = field_schema["$ref"].rsplit("/", 1)[-1]
|
|
795
|
-
fields.append({"name": field_name, "type": kind, "required": field_name in required})
|
|
796
716
|
entities.append(
|
|
797
717
|
{
|
|
798
718
|
"name": name,
|
|
799
719
|
"primaryKey": list(entity.get("primary_key") or []) if isinstance(entity, dict) else [],
|
|
800
720
|
"unique": list(entity.get("unique") or []) if isinstance(entity, dict) else [],
|
|
801
721
|
"relationships": list(entity.get("relationships") or []) if isinstance(entity, dict) else [],
|
|
802
|
-
"fields":
|
|
722
|
+
"fields": [asdict(field) for field in fields_by_entity.get(name, [])],
|
|
803
723
|
}
|
|
804
724
|
)
|
|
805
725
|
_annotate_redaction(entities, policy)
|
|
@@ -860,8 +780,9 @@ def _describe(request: dict) -> dict:
|
|
|
860
780
|
template by slug, or a template's files), so the page shows what a world gets;
|
|
861
781
|
a compiled world's own files (``schema/world.json`` + ``tool-schemas.json``) pass
|
|
862
782
|
straight through. Never runs the connector's Python."""
|
|
863
|
-
from .connector import parse_spec
|
|
783
|
+
from .connector import load_spec, parse_spec
|
|
864
784
|
from .redaction import env_policy
|
|
785
|
+
from .ui import ui_kind
|
|
865
786
|
from .scaffold import (
|
|
866
787
|
TEMPLATES,
|
|
867
788
|
init_bundle,
|
|
@@ -883,7 +804,7 @@ def _describe(request: dict) -> dict:
|
|
|
883
804
|
if path.is_file() and "__pycache__" not in path.parts
|
|
884
805
|
}
|
|
885
806
|
else:
|
|
886
|
-
authored =
|
|
807
|
+
authored = request_files(request)
|
|
887
808
|
if all(marker in authored for marker in COMPILED_MARKERS):
|
|
888
809
|
root.mkdir()
|
|
889
810
|
for path, data in authored.items():
|
|
@@ -899,6 +820,7 @@ def _describe(request: dict) -> dict:
|
|
|
899
820
|
summary = _template_summary({"world.json": (root / "schema/world.json").read_bytes()})["summary"]
|
|
900
821
|
summary["digest"] = template_digest(authored)
|
|
901
822
|
summary["files"] = sorted(authored)
|
|
823
|
+
summary["ui"] = ui_kind(root)
|
|
902
824
|
tool_schemas = {}
|
|
903
825
|
if (root / "tool-schemas.json").is_file():
|
|
904
826
|
tool_schemas = json.loads((root / "tool-schemas.json").read_text(encoding="utf-8")).get("tools") or {}
|
|
@@ -916,7 +838,7 @@ def _describe(request: dict) -> dict:
|
|
|
916
838
|
if (root / "connector.toml").is_file():
|
|
917
839
|
from .api import load_api_settings, openapi_document
|
|
918
840
|
|
|
919
|
-
spec =
|
|
841
|
+
spec = load_spec(root)
|
|
920
842
|
try:
|
|
921
843
|
settings = load_api_settings(root, spec)
|
|
922
844
|
except ContractError:
|
|
@@ -953,52 +875,6 @@ def _describe(request: dict) -> dict:
|
|
|
953
875
|
return {"status": "ok", "description": summary}
|
|
954
876
|
|
|
955
877
|
|
|
956
|
-
def _import_data(request: dict) -> dict:
|
|
957
|
-
"""``import-data``: rows into a world the platform holds as files. The tree arrives
|
|
958
|
-
as ``[{path, content}]``, the rows as ``{entity: [row, …]}`` with a mode; the world is
|
|
959
|
-
rebuilt in a temporary directory, the rows go in through :func:`data.import_rows`
|
|
960
|
-
(the same gate the CLI uses, scratch copy first), and every file of the updated
|
|
961
|
-
tree comes back the way ``init-template`` returns one, so the platform commits a
|
|
962
|
-
version from it without knowing which file changed."""
|
|
963
|
-
from .data import KIND_SCHEMA, MERGES, MODES, import_rows
|
|
964
|
-
from .redaction import parse_mode
|
|
965
|
-
|
|
966
|
-
rows = request.get("rows")
|
|
967
|
-
mode = request.get("mode", "append")
|
|
968
|
-
merge = request.get("merge", "upsert")
|
|
969
|
-
if not isinstance(rows, dict) or not all(isinstance(v, list) for v in rows.values()):
|
|
970
|
-
fail("input", "Expected rows as {entity: [row, …]}")
|
|
971
|
-
if mode not in MODES:
|
|
972
|
-
fail("input", f"mode must be one of {', '.join(MODES)}")
|
|
973
|
-
if merge not in MERGES:
|
|
974
|
-
fail("input", f"merge must be one of {', '.join(MERGES)}")
|
|
975
|
-
redact = parse_mode(request.get("redact"))
|
|
976
|
-
dry_run = _flag(request, "dry_run", False)
|
|
977
|
-
with tempfile.TemporaryDirectory(prefix="gateway-import-") as directory:
|
|
978
|
-
root = Path(directory) / "bundle"
|
|
979
|
-
root.mkdir()
|
|
980
|
-
_write_files(root, request.get("files"))
|
|
981
|
-
world = world_root(root)
|
|
982
|
-
result = import_rows(world, rows, mode=mode, merge=merge, redact=redact, dry_run=dry_run)
|
|
983
|
-
keys = ("kind", "entity_counts", "changed")
|
|
984
|
-
report = {"status": "ok", **{key: result[key] for key in keys}}
|
|
985
|
-
if dry_run:
|
|
986
|
-
return {**report, **{key: result[key] for key in ("proof", "deferred") if key in result}}
|
|
987
|
-
if result["kind"] == KIND_SCHEMA and snapshots.default_snapshot(world).is_file():
|
|
988
|
-
# The journals grew; a snapshot built before this import would be
|
|
989
|
-
# served over them. Rebuild so the tree that travels back is one thing.
|
|
990
|
-
from .compiler import build
|
|
991
|
-
from .snapshot import build_all
|
|
992
|
-
|
|
993
|
-
build_all(world, build(world)) # build(): the contract, no snapshot audit
|
|
994
|
-
if world != root and result["kind"] == KIND_SCHEMA:
|
|
995
|
-
# The bundle's base seed is derived from the world's data; older runners post it.
|
|
996
|
-
from .api import write_base_seed
|
|
997
|
-
|
|
998
|
-
write_base_seed(world, root)
|
|
999
|
-
return {**report, "files": _tree(root)}
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
878
|
def _ingest_calls(request: dict) -> dict:
|
|
1003
879
|
"""``ingest-calls``: records into a world the platform holds as files — the twin
|
|
1004
880
|
of ``import-data`` for the record contract of :mod:`.ingest`. The tree arrives
|
|
@@ -1252,7 +1128,7 @@ def _api_bundle(request: dict) -> dict:
|
|
|
1252
1128
|
what `gateway worlds schema api-bundle` writes locally, for a CLI with no Python."""
|
|
1253
1129
|
from .api import build_api_bundle
|
|
1254
1130
|
|
|
1255
|
-
files =
|
|
1131
|
+
files = request_files(request)
|
|
1256
1132
|
with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
|
|
1257
1133
|
root = Path(directory) / "world"
|
|
1258
1134
|
root.mkdir()
|
|
@@ -1364,11 +1240,29 @@ def _run_tests(request: dict) -> dict:
|
|
|
1364
1240
|
return {"status": "ok", "report": report}
|
|
1365
1241
|
|
|
1366
1242
|
|
|
1243
|
+
def _prepared_tree(request: dict) -> Path:
|
|
1244
|
+
"""The tree the platform laid under a ``workdir`` it created — ``tree``, a
|
|
1245
|
+
relative path under it, beside any snapshot it checked out — opened in place.
|
|
1246
|
+
Every path is validated against the workdir (:func:`_under`); the workdir is
|
|
1247
|
+
the caller's to remove."""
|
|
1248
|
+
if request.get("files") is not None or request.get("world") is not None:
|
|
1249
|
+
fail("input", "a request opens its world through one of workdir, files or world")
|
|
1250
|
+
workdir = _workdir(request)
|
|
1251
|
+
tree = _under(workdir, request.get("tree"), "tree")
|
|
1252
|
+
if not tree.is_dir():
|
|
1253
|
+
fail("input", "tree must be a directory under workdir")
|
|
1254
|
+
return world_root(tree)
|
|
1255
|
+
|
|
1256
|
+
|
|
1367
1257
|
def _world_for(request: dict, staging: Path) -> Path:
|
|
1368
1258
|
"""The world a run-style command works on: ``files`` laid into ``staging``
|
|
1369
|
-
(what the platform sends
|
|
1370
|
-
|
|
1371
|
-
``
|
|
1259
|
+
(what the platform sends for a small tree), the tree under a ``workdir`` the
|
|
1260
|
+
platform prepared (:func:`_prepared_tree`, for a large one), or — where this
|
|
1261
|
+
runtime may run authored code — ``world``, an absolute directory on this
|
|
1262
|
+
machine, opened in place so its ``[dependencies]`` paths resolve and nothing
|
|
1263
|
+
is copied."""
|
|
1264
|
+
if request.get("workdir") is not None:
|
|
1265
|
+
return _prepared_tree(request)
|
|
1372
1266
|
world = request.get("world")
|
|
1373
1267
|
if world is None:
|
|
1374
1268
|
root = staging / "world"
|
|
@@ -1492,109 +1386,12 @@ def _dry_run(request: dict) -> dict:
|
|
|
1492
1386
|
return answer
|
|
1493
1387
|
|
|
1494
1388
|
|
|
1495
|
-
def _counts(request: dict) -> dict:
|
|
1496
|
-
"""``counts``: what the world at these files seeds with. A kind whose seed lives
|
|
1497
|
-
elsewhere (a harbor tree, an unknown layout) answers with the reason instead of
|
|
1498
|
-
a rejection, so the caller can show it."""
|
|
1499
|
-
from .data import entity_counts, entity_rows, world_kind
|
|
1500
|
-
|
|
1501
|
-
entity = request.get("entity")
|
|
1502
|
-
if entity is not None and not isinstance(entity, str):
|
|
1503
|
-
fail("input", "entity must be a string")
|
|
1504
|
-
with tempfile.TemporaryDirectory(prefix="gateway-counts-") as directory:
|
|
1505
|
-
root = Path(directory) / "world"
|
|
1506
|
-
root.mkdir()
|
|
1507
|
-
_write_files(root, request.get("files"))
|
|
1508
|
-
root = world_root(root)
|
|
1509
|
-
kind = world_kind(root)
|
|
1510
|
-
try:
|
|
1511
|
-
counts = entity_counts(root)
|
|
1512
|
-
except ContractError as error:
|
|
1513
|
-
if error.detail.get("code") != "kind":
|
|
1514
|
-
raise
|
|
1515
|
-
return {
|
|
1516
|
-
"status": "ok",
|
|
1517
|
-
"kind": kind,
|
|
1518
|
-
"entity_counts": None,
|
|
1519
|
-
"reason": error.detail["message"],
|
|
1520
|
-
}
|
|
1521
|
-
report = {"status": "ok", "kind": kind, "entity_counts": counts}
|
|
1522
|
-
if entity is not None:
|
|
1523
|
-
report["rows"] = entity_rows(root, entity)
|
|
1524
|
-
return report
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
def _query(request: dict) -> dict:
|
|
1528
|
-
"""``query``: the rows of one entity of the world at these files that match
|
|
1529
|
-
``where`` (a field to a value or a list of values), a page at a time (``limit``,
|
|
1530
|
-
``offset``). With ``resolve``, each value is plaintext and is looked up in the
|
|
1531
|
-
forms the world's ``[redact]`` policy stores (:func:`data.entity_query`). A tree
|
|
1532
|
-
that carries its compiled ``db/state.sqlite`` (a checked-out snapshot) answers
|
|
1533
|
-
from the snapshot, as ``counts`` does; otherwise from its data files. With
|
|
1534
|
-
``across`` (local runtime only: ``world`` is a directory here, so its
|
|
1535
|
-
``[dependencies]`` paths resolve) the answer is :func:`data.entity_query_across`'s
|
|
1536
|
-
groups, one per linked world a mapping carries a ``where`` field into."""
|
|
1537
|
-
from .data import entity_query, entity_query_across, world_kind
|
|
1538
|
-
|
|
1539
|
-
entity = request.get("entity")
|
|
1540
|
-
if not isinstance(entity, str) or not entity:
|
|
1541
|
-
fail("input", "entity must be a non-empty string")
|
|
1542
|
-
where = request.get("where", {})
|
|
1543
|
-
if not isinstance(where, dict):
|
|
1544
|
-
fail("input", "where must be an object mapping a field to a value or a list of values")
|
|
1545
|
-
resolve = request.get("resolve", False)
|
|
1546
|
-
if not isinstance(resolve, bool):
|
|
1547
|
-
fail("input", "resolve must be a boolean")
|
|
1548
|
-
# The platform's cross-world query sends the mapping's transform chain per field
|
|
1549
|
-
# under this key (worldDataQueryService.ts BRIDGE_WHERE_TRANSFORM_KEY).
|
|
1550
|
-
where_transform = request.get("where_transform")
|
|
1551
|
-
if where_transform is not None and not isinstance(where_transform, dict):
|
|
1552
|
-
fail("input", "where_transform must be an object mapping a field to a list of transform names")
|
|
1553
|
-
across = request.get("across", False)
|
|
1554
|
-
compare = request.get("compare", False)
|
|
1555
|
-
if not isinstance(across, bool) or not isinstance(compare, bool):
|
|
1556
|
-
fail("input", "across and compare must be booleans")
|
|
1557
|
-
if compare and not across:
|
|
1558
|
-
fail("input", "compare needs across")
|
|
1559
|
-
scenario = request.get("scenario")
|
|
1560
|
-
if scenario is not None and (not isinstance(scenario, str) or not scenario):
|
|
1561
|
-
fail("input", "scenario must be a non-empty string", "/scenario")
|
|
1562
|
-
with tempfile.TemporaryDirectory(prefix="gateway-query-") as directory:
|
|
1563
|
-
root = _world_for(request, Path(directory))
|
|
1564
|
-
if across:
|
|
1565
|
-
if request.get("world") is None:
|
|
1566
|
-
fail(
|
|
1567
|
-
"input", "across reads the linked worlds at their paths, so it takes world (a directory), not files"
|
|
1568
|
-
)
|
|
1569
|
-
report = entity_query_across(
|
|
1570
|
-
root,
|
|
1571
|
-
entity,
|
|
1572
|
-
where,
|
|
1573
|
-
limit=request.get("limit", 100),
|
|
1574
|
-
offset=request.get("offset", 0),
|
|
1575
|
-
resolve=resolve,
|
|
1576
|
-
compare=compare,
|
|
1577
|
-
scenario=scenario,
|
|
1578
|
-
)
|
|
1579
|
-
return {"status": "ok", "kind": world_kind(root), **report}
|
|
1580
|
-
report = entity_query(
|
|
1581
|
-
root,
|
|
1582
|
-
entity,
|
|
1583
|
-
where,
|
|
1584
|
-
limit=request.get("limit", 100),
|
|
1585
|
-
offset=request.get("offset", 0),
|
|
1586
|
-
resolve=resolve,
|
|
1587
|
-
where_transform=where_transform,
|
|
1588
|
-
scenario=scenario,
|
|
1589
|
-
)
|
|
1590
|
-
return {"status": "ok", "kind": world_kind(root), **report}
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
1389
|
def _templates(request: dict) -> dict:
|
|
1594
1390
|
"""``templates``: every installed connector template, read from its ``world.json``
|
|
1595
1391
|
(name and description when the document carries them) and whether it declares a
|
|
1596
1392
|
vendor connection. Never runs a template's Python."""
|
|
1597
1393
|
from .scaffold import TEMPLATES, list_templates
|
|
1394
|
+
from .ui import ui_kind
|
|
1598
1395
|
|
|
1599
1396
|
templates = []
|
|
1600
1397
|
for slug in list_templates():
|
|
@@ -1610,6 +1407,7 @@ def _templates(request: dict) -> dict:
|
|
|
1610
1407
|
"entities": sorted(gateway.get("entities") or {}),
|
|
1611
1408
|
"tools": sorted(gateway.get("tools") or {}),
|
|
1612
1409
|
"hasConnection": (TEMPLATES / slug / "connector.toml").is_file(),
|
|
1410
|
+
"ui": ui_kind(TEMPLATES / slug),
|
|
1613
1411
|
}
|
|
1614
1412
|
)
|
|
1615
1413
|
return {"status": "ok", "templates": templates}
|
|
@@ -1630,7 +1428,11 @@ def execute(request: dict) -> dict:
|
|
|
1630
1428
|
if request["command"] == "capture":
|
|
1631
1429
|
return _capture(request)
|
|
1632
1430
|
if request["command"] == "import-data":
|
|
1633
|
-
return
|
|
1431
|
+
return platform_import.import_data(request)
|
|
1432
|
+
if request["command"] == "import-files":
|
|
1433
|
+
return platform_import.import_files(request, _world_for)
|
|
1434
|
+
if request["command"] == "chunk-files":
|
|
1435
|
+
return platform_import.chunk_files(request, _world_for)
|
|
1634
1436
|
if request["command"] == "ingest":
|
|
1635
1437
|
return _ingest(request)
|
|
1636
1438
|
if request["command"] == "ingest-calls":
|
|
@@ -1646,15 +1448,15 @@ def execute(request: dict) -> dict:
|
|
|
1646
1448
|
if request["command"] == "dry-run":
|
|
1647
1449
|
return _dry_run(request)
|
|
1648
1450
|
if request["command"] == "counts":
|
|
1649
|
-
return
|
|
1451
|
+
return platform_query.counts(request, _world_for)
|
|
1650
1452
|
if request["command"] == "query":
|
|
1651
|
-
return
|
|
1453
|
+
return platform_query.query(request, _world_for)
|
|
1652
1454
|
if request["command"] == "templates":
|
|
1653
1455
|
return _templates(request)
|
|
1654
1456
|
if request["command"] == "api-bundle":
|
|
1655
1457
|
return _api_bundle(request)
|
|
1656
1458
|
if request["command"] == "template-summary":
|
|
1657
|
-
return _template_summary(
|
|
1459
|
+
return _template_summary(request_files(request))
|
|
1658
1460
|
if request["command"] == "describe":
|
|
1659
1461
|
return _describe(request)
|
|
1660
1462
|
extra: dict = {}
|
|
@@ -1723,7 +1525,14 @@ def execute(request: dict) -> dict:
|
|
|
1723
1525
|
name = request.get("name")
|
|
1724
1526
|
if not isinstance(connector, str) or (name is not None and not isinstance(name, str)):
|
|
1725
1527
|
fail("input", "Expected the connector slug and an optional name")
|
|
1726
|
-
init_bundle_from_files(root,
|
|
1528
|
+
init_bundle_from_files(root, request_files(request), connector=connector, name=name)
|
|
1529
|
+
if request["command"] == "init-openapi":
|
|
1530
|
+
from .openapi_import import OpenApiImport
|
|
1531
|
+
|
|
1532
|
+
planned = OpenApiImport.execute(request, root)
|
|
1533
|
+
if not planned["written"]:
|
|
1534
|
+
return planned
|
|
1535
|
+
extra.update(planned)
|
|
1727
1536
|
if request["command"] == "build-slack":
|
|
1728
1537
|
from .slack import build_slack_bundle
|
|
1729
1538
|
|
|
@@ -1737,13 +1546,15 @@ def execute(request: dict) -> dict:
|
|
|
1737
1546
|
**extra,
|
|
1738
1547
|
}
|
|
1739
1548
|
if request["command"] not in ("validate", "validate-archive"):
|
|
1549
|
+
paths = [path for path in sorted(root.rglob("*")) if path.is_file()]
|
|
1550
|
+
if len(paths) > MAX_FILES:
|
|
1551
|
+
fail("size", f"The world tree has {len(paths)} files, over the {MAX_FILES} limit")
|
|
1740
1552
|
report["files"] = [
|
|
1741
1553
|
{
|
|
1742
1554
|
"path": str(path.relative_to(root)),
|
|
1743
1555
|
"content": base64.b64encode(path.read_bytes()).decode("ascii"),
|
|
1744
1556
|
}
|
|
1745
|
-
for path in
|
|
1746
|
-
if path.is_file()
|
|
1557
|
+
for path in paths
|
|
1747
1558
|
]
|
|
1748
1559
|
return report
|
|
1749
1560
|
|