@withgateway/sdk 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +24 -8
  2. package/dist/cli.js +15468 -11263
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +15233 -11028
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +291 -40
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +284 -37
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +203 -47
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +198 -42
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
  19. package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
  20. package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
  21. package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
  22. package/dist/index.d.mts +3 -3
  23. package/dist/index.d.ts +3 -3
  24. package/dist/index.js +2149 -615
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2053 -566
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/traces/index.js +268 -31
  29. package/dist/traces/index.js.map +1 -1
  30. package/dist/traces/index.mjs +262 -28
  31. package/dist/traces/index.mjs.map +1 -1
  32. package/dist/tracing/index.js +19 -11
  33. package/dist/tracing/index.js.map +1 -1
  34. package/dist/tracing/index.mjs +19 -11
  35. package/dist/tracing/index.mjs.map +1 -1
  36. package/dist/worlds/index.d.mts +886 -117
  37. package/dist/worlds/index.d.ts +886 -117
  38. package/dist/worlds/index.js +1770 -452
  39. package/dist/worlds/index.js.map +1 -1
  40. package/dist/worlds/index.mjs +1752 -434
  41. package/dist/worlds/index.mjs.map +1 -1
  42. package/package.json +14 -10
  43. package/runtime/gatewaysdk/environment/_toml.py +3 -10
  44. package/runtime/gatewaysdk/environment/_world.py +95 -28
  45. package/runtime/gatewaysdk/environment/runtime.py +10 -6
  46. package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
  47. package/runtime/gatewaysdk/environment/schema/api.py +19 -23
  48. package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
  49. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
  50. package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
  51. package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
  52. package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
  53. package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
  54. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
  55. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
  56. package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
  57. package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
  58. package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
  59. package/runtime/gatewaysdk/environment/schema/data.py +21 -5
  60. package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
  61. package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
  62. package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
  63. package/runtime/gatewaysdk/environment/schema/host.py +68 -115
  64. package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
  65. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
  66. package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
  67. package/runtime/gatewaysdk/environment/schema/local.py +7 -10
  68. package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
  69. package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
  70. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
  71. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
  72. package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
  73. package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
  74. package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
  75. package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
  76. package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
  77. package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
  78. package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
  79. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
  80. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
  81. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
  82. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
  83. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
  84. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
  85. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  86. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
  87. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  88. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
  89. package/runtime/gatewaysdk/environment/schema/store.py +15 -6
  90. package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
  91. package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
  92. package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
  93. package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
  94. package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
  95. package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
  96. package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
  97. package/runtime/gatewaysdk/environment/world.py +42 -3
  98. package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
  99. package/skills/benchmarks-getting-started/SKILL.md +70 -22
  100. package/skills/connector-schema-authoring/SKILL.md +54 -4
  101. package/skills/connector-schema-authoring/connector-worlds.md +21 -7
  102. package/skills/connector-schema-authoring/world-schema.md +144 -83
  103. package/skills/hooks-getting-started/SKILL.md +27 -2
  104. package/skills/world-data-ingestion/ROWS.md +62 -0
  105. package/skills/world-data-ingestion/SKILL.md +91 -54
  106. package/skills/worlds-getting-started/SKILL.md +146 -146
  107. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  108. package/skills/worlds-getting-started/references/gotchas.md +30 -0
  109. package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  110. package/skills/worlds-getting-started/references/openapi-import.md +90 -0
@@ -5,11 +5,14 @@ inside the hosted runner or an isolated workbench, never the publication server.
5
5
  The stdio protocol deliberately contains bundle bytes and mapping context only.
6
6
  """
7
7
 
8
+ # file limit exception: the bridge's command dispatcher and its import/snapshot/
9
+ # check handlers share one request-validation vocabulary (_workdir, _under,
10
+ # _write_tree, fail); the read commands already live in platform_query.py and
11
+ # the next split is the import/snapshot half. This file must not grow.
12
+
8
13
  from __future__ import annotations
9
14
 
10
- import ast
11
15
  import base64
12
- import binascii
13
16
  import gzip
14
17
  import io
15
18
  import json
@@ -18,15 +21,19 @@ import shutil
18
21
  import sys
19
22
  import tarfile
20
23
  import tempfile
24
+ from dataclasses import asdict
21
25
  from pathlib import Path, PurePosixPath
22
26
  from typing import TYPE_CHECKING, Any
23
27
 
24
28
  from .._materialize import materialize_template
25
29
  from ..world import WorldTree
26
- from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle
30
+ from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle, platform_import, platform_query
27
31
  from . import snapshot as snapshots
32
+ from .cast import world_fields
28
33
  from .compiler import retarget_empty_journals
34
+ from .handler_scan import HandlerScan
29
35
  from .redaction import check_world_redaction
36
+ from .scaffold import request_files
30
37
  from .validation import (
31
38
  digest,
32
39
  encoded,
@@ -50,6 +57,7 @@ COMMANDS = (
50
57
  "validate-archive",
51
58
  "init-template",
52
59
  "init-template-files",
60
+ "init-openapi",
53
61
  "template-summary",
54
62
  "describe",
55
63
  "compile",
@@ -58,6 +66,8 @@ COMMANDS = (
58
66
  "refresh-slack",
59
67
  "capture",
60
68
  "import-data",
69
+ "import-files",
70
+ "chunk-files",
61
71
  "ingest",
62
72
  "ingest-calls",
63
73
  "conform",
@@ -238,62 +248,7 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
238
248
  fail("mode", "Expected a schema world: gateway-env.toml with [schema] beside schema/world.json")
239
249
  contract = check_bundle(bundle)
240
250
  declared = set(contract.tools)
241
- found: dict[str, str] = {}
242
- for path in sorted((bundle / "tools").glob("*.py")):
243
- if path.is_symlink():
244
- fail("path", "Tool handlers must be regular files")
245
- try:
246
- tree = ast.parse(path.read_text(encoding="utf-8"), filename=path.name)
247
- except (SyntaxError, UnicodeError):
248
- fail("handler", f"Invalid Python handler syntax: {path.name}")
249
- for node in tree.body:
250
- if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
251
- continue
252
- names = []
253
- for decorator in node.decorator_list:
254
- target = decorator.func if isinstance(decorator, ast.Call) else decorator
255
- is_tool = (isinstance(target, ast.Name) and target.id == "tool") or (
256
- isinstance(target, ast.Attribute) and target.attr == "tool"
257
- )
258
- if not is_tool:
259
- continue
260
- name = node.name
261
- if isinstance(decorator, ast.Call):
262
- for keyword in decorator.keywords:
263
- if keyword.arg == "name":
264
- if not isinstance(keyword.value, ast.Constant) or not isinstance(keyword.value.value, str):
265
- fail("handler", "Publication requires literal tool names")
266
- name = keyword.value.value
267
- names.append(name)
268
- if not names and node.name in declared:
269
- names.append(node.name)
270
- if not names and node.name == "run" and path.stem in declared:
271
- names.append(path.stem)
272
- for name in names:
273
- if name in found:
274
- fail("handler", f"Duplicate handler for {name}")
275
- found[name] = path.name
276
- if set(found) != declared:
277
- undeclared = sorted(set(found) - declared)
278
- unhandled = sorted(declared - set(found))
279
- parts = []
280
- if unhandled:
281
- parts.append(
282
- "declared in schema/world.json (tools) without a handler under tools/: "
283
- + ", ".join(unhandled)
284
- + " — add `@tool` def <name>(...) to a tools/*.py, or drop the declaration"
285
- )
286
- if undeclared:
287
- parts.append(
288
- "handled under tools/ without a declaration: "
289
- + ", ".join(f"{name} ({found[name]})" for name in undeclared)
290
- + " — declare each in schema/world.json, or remove the handler"
291
- )
292
- fail(
293
- "handler",
294
- "Declared tools and Python handlers must match exactly; " + "; ".join(parts),
295
- "/tools",
296
- )
251
+ HandlerScan(bundle, declared).run()
297
252
  if (bundle / "connector.toml").is_file():
298
253
  from .api import load_api_settings
299
254
  from .connector import parse_spec
@@ -393,12 +348,10 @@ def _check_verifiers(bundle: Path, store: SQLiteStore, unattached: dict[str, str
393
348
  """Every ``verifiers/*.sql`` run over the template through the store's
394
349
  read-only authorizer, the way a session grades. A verifier that names a
395
350
  table or column the contract lacks is refused here — at check — not at
396
- the first grade. ``scored`` reports the score over the initial state. A
397
- verifier that reads a dependency the check could not attach (a pin
398
- without a local path) is reported, not refused: it runs where the
399
- platform opens the link."""
400
- from .._world import split_sql_statements
401
-
351
+ the first grade. ``scored`` reports the score over the initial state by
352
+ the one :class:`SqlVerifier` rule a grade applies. A verifier that reads
353
+ a dependency the check could not attach (a pin without a local path) is
354
+ reported, not refused: it runs where the platform opens the link."""
402
355
  verifiers_dir = bundle / "verifiers"
403
356
  if not verifiers_dir.is_dir():
404
357
  return {}
@@ -406,42 +359,48 @@ def _check_verifiers(bundle: Path, store: SQLiteStore, unattached: dict[str, str
406
359
  for path in sorted(verifiers_dir.glob("*.sql")):
407
360
  if path.is_symlink():
408
361
  fail("path", "Verifiers must be regular files", f"/verifiers/{path.name}")
409
- score: Any = None
410
- needs: str | None = None
411
- for statement in split_sql_statements(path.read_text(encoding="utf-8")):
412
- try:
413
- rows = store.query(statement, private=True)
414
- except ContractError as error:
415
- reason = error.__cause__ if error.__cause__ is not None else error.detail["message"]
416
- alias = _missing_dependency_alias(str(reason), unattached)
417
- if alias is not None:
418
- needs = alias
419
- break
420
- hint = ""
421
- if "no such table" in str(reason):
422
- tables = sorted(store.contract.entities)
423
- tables += [f"{a}.{t}" for a, ts in sorted(store.attached.items()) for t in sorted(ts)]
424
- hint = f"; the world's entities are {', '.join(tables) or '(none)'}"
425
- fail(
426
- "verifier",
427
- f"verifiers/{path.name} does not run against this schema: {reason}{hint}",
428
- f"/verifiers/{path.name}",
429
- )
430
- if rows:
431
- score = rows[0][0]
432
- if needs is not None:
433
- out[path.stem] = {
434
- "status": "needs_dependency",
435
- "dependency": needs,
436
- "reason": f"reads {needs}.<table>; {unattached[needs]}",
437
- }
438
- elif scored:
439
- out[path.stem] = {"status": "ok", "initial_score": score}
440
- else:
441
- out[path.stem] = {"status": "ok"}
362
+ out[path.stem] = _check_verifier(path, store, unattached, scored=scored)
442
363
  return out
443
364
 
444
365
 
366
+ def _check_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str], *, scored: bool) -> dict[str, Any]:
367
+ from .._world import SqlVerifier, VerifierError
368
+
369
+ try:
370
+ score, _sql = SqlVerifier.read(path).score(lambda statement: store.query(statement, private=True))
371
+ except ContractError as error:
372
+ alias = _refuse_verifier(path, store, unattached, error)
373
+ return {
374
+ "status": "needs_dependency",
375
+ "dependency": alias,
376
+ "reason": f"reads {alias}.<table>; {unattached[alias]}",
377
+ }
378
+ except VerifierError as error:
379
+ fail("verifier", str(error), f"/verifiers/{path.name}")
380
+ if not scored:
381
+ return {"status": "ok"}
382
+ return {"status": "ok", "initial_score": score}
383
+
384
+
385
+ def _refuse_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str], error: ContractError) -> str:
386
+ """The unattached dependency alias the failure names — the one refusal
387
+ check reports rather than raises; every other failure is refused here."""
388
+ reason = error.__cause__ if error.__cause__ is not None else error.detail["message"]
389
+ alias = _missing_dependency_alias(str(reason), unattached)
390
+ if alias is not None:
391
+ return alias
392
+ hint = ""
393
+ if "no such table" in str(reason):
394
+ tables = sorted(store.contract.entities)
395
+ tables += [f"{a}.{t}" for a, ts in sorted(store.attached.items()) for t in sorted(ts)]
396
+ hint = f"; the world's entities are {', '.join(tables) or '(none)'}"
397
+ fail(
398
+ "verifier",
399
+ f"verifiers/{path.name} does not run against this schema: {reason}{hint}",
400
+ f"/verifiers/{path.name}",
401
+ )
402
+
403
+
445
404
  def _missing_dependency_alias(reason: str, unattached: dict[str, str]) -> str | None:
446
405
  """The alias of an unattached dependency a ``no such table: alias.table``
447
406
  failure names, else ``None``."""
@@ -698,25 +657,6 @@ def _capture(request: dict) -> dict:
698
657
  }
699
658
 
700
659
 
701
- def _template_files(request: dict) -> dict[str, bytes]:
702
- listed = request.get("files")
703
- if not isinstance(listed, list) or not listed:
704
- fail("input", "Expected the template's files as [{path, content}]")
705
- files: dict[str, bytes] = {}
706
- for entry in listed:
707
- if (
708
- not isinstance(entry, dict)
709
- or not isinstance(entry.get("path"), str)
710
- or not isinstance(entry.get("content"), str)
711
- ):
712
- fail("input", "Each template file is {path, content} with base64 content")
713
- try:
714
- files[entry["path"]] = base64.b64decode(entry["content"], validate=True)
715
- except (ValueError, binascii.Error):
716
- fail("input", "Template file content must be base64")
717
- return files
718
-
719
-
720
660
  def _redact_summary(spec: "Spec | RedactPolicy") -> dict:
721
661
  """The ``[redact]`` table as describe reports it, so a caller can compute a
722
662
  stored form from these lists without running Python. ``slug`` is the SALT —
@@ -748,7 +688,7 @@ def _template_summary(files: dict[str, bytes]) -> dict:
748
688
  """What a workspace connector is, read from its files: the vendor connection
749
689
  (connector.toml) and the data model (world.json), plus the digest the platform
750
690
  pins worlds to. Never runs the template's Python."""
751
- from .connector import parse_spec
691
+ from .connector import bind_spec, parse_spec
752
692
  from .redaction import parse_env_policy
753
693
  from .scaffold import template_digest
754
694
 
@@ -766,40 +706,20 @@ def _template_summary(files: dict[str, bytes]) -> dict:
766
706
  gateway = world.get("x-gateway") if isinstance(world, dict) else None
767
707
  if not isinstance(gateway, dict):
768
708
  fail("template", f"{world_path} is not a gateway-world document")
769
- spec = parse_spec(files["connector.toml"].decode("utf-8")) if "connector.toml" in files else None
709
+ spec = bind_spec(parse_spec(files["connector.toml"].decode("utf-8")), world) if "connector.toml" in files else None
770
710
  policy = spec
771
711
  if policy is None and "gateway-env.toml" in files:
772
712
  policy = parse_env_policy(files["gateway-env.toml"].decode("utf-8"))
773
713
  entities = []
714
+ fields_by_entity = world_fields(world)
774
715
  for name, entity in sorted((gateway.get("entities") or {}).items()):
775
- fields: list = []
776
- schema = entity.get("schema") if isinstance(entity, dict) else None
777
- ref = schema.get("$ref") if isinstance(schema, dict) else None
778
- definition = None
779
- if isinstance(ref, str) and ref.startswith("#/$defs/"):
780
- definition = (world.get("$defs") or {}).get(ref[len("#/$defs/") :])
781
- elif isinstance(schema, dict) and isinstance(schema.get("properties"), dict):
782
- # An inline payload (`schema entity add --schema '{…}'`) declares its
783
- # fields right here; it is as much a field list as a $defs entry.
784
- definition = schema
785
- properties = (definition or {}).get("properties") or {} if isinstance(definition, dict) else {}
786
- required = set((definition or {}).get("required") or []) if isinstance(definition, dict) else set()
787
- for field_name, field_schema in properties.items():
788
- kind = field_schema.get("type") if isinstance(field_schema, dict) else None
789
- if isinstance(kind, list):
790
- # A nullable field is `["string", "null"]` in JSON Schema; the page
791
- # shows it as one readable type, not the list's concatenation.
792
- kind = " | ".join(str(part) for part in kind)
793
- if kind is None and isinstance(field_schema, dict) and isinstance(field_schema.get("$ref"), str):
794
- kind = field_schema["$ref"].rsplit("/", 1)[-1]
795
- fields.append({"name": field_name, "type": kind, "required": field_name in required})
796
716
  entities.append(
797
717
  {
798
718
  "name": name,
799
719
  "primaryKey": list(entity.get("primary_key") or []) if isinstance(entity, dict) else [],
800
720
  "unique": list(entity.get("unique") or []) if isinstance(entity, dict) else [],
801
721
  "relationships": list(entity.get("relationships") or []) if isinstance(entity, dict) else [],
802
- "fields": fields,
722
+ "fields": [asdict(field) for field in fields_by_entity.get(name, [])],
803
723
  }
804
724
  )
805
725
  _annotate_redaction(entities, policy)
@@ -860,8 +780,9 @@ def _describe(request: dict) -> dict:
860
780
  template by slug, or a template's files), so the page shows what a world gets;
861
781
  a compiled world's own files (``schema/world.json`` + ``tool-schemas.json``) pass
862
782
  straight through. Never runs the connector's Python."""
863
- from .connector import parse_spec
783
+ from .connector import load_spec, parse_spec
864
784
  from .redaction import env_policy
785
+ from .ui import ui_kind
865
786
  from .scaffold import (
866
787
  TEMPLATES,
867
788
  init_bundle,
@@ -883,7 +804,7 @@ def _describe(request: dict) -> dict:
883
804
  if path.is_file() and "__pycache__" not in path.parts
884
805
  }
885
806
  else:
886
- authored = _template_files(request)
807
+ authored = request_files(request)
887
808
  if all(marker in authored for marker in COMPILED_MARKERS):
888
809
  root.mkdir()
889
810
  for path, data in authored.items():
@@ -899,6 +820,7 @@ def _describe(request: dict) -> dict:
899
820
  summary = _template_summary({"world.json": (root / "schema/world.json").read_bytes()})["summary"]
900
821
  summary["digest"] = template_digest(authored)
901
822
  summary["files"] = sorted(authored)
823
+ summary["ui"] = ui_kind(root)
902
824
  tool_schemas = {}
903
825
  if (root / "tool-schemas.json").is_file():
904
826
  tool_schemas = json.loads((root / "tool-schemas.json").read_text(encoding="utf-8")).get("tools") or {}
@@ -916,7 +838,7 @@ def _describe(request: dict) -> dict:
916
838
  if (root / "connector.toml").is_file():
917
839
  from .api import load_api_settings, openapi_document
918
840
 
919
- spec = parse_spec((root / "connector.toml").read_text(encoding="utf-8"))
841
+ spec = load_spec(root)
920
842
  try:
921
843
  settings = load_api_settings(root, spec)
922
844
  except ContractError:
@@ -953,52 +875,6 @@ def _describe(request: dict) -> dict:
953
875
  return {"status": "ok", "description": summary}
954
876
 
955
877
 
956
- def _import_data(request: dict) -> dict:
957
- """``import-data``: rows into a world the platform holds as files. The tree arrives
958
- as ``[{path, content}]``, the rows as ``{entity: [row, …]}`` with a mode; the world is
959
- rebuilt in a temporary directory, the rows go in through :func:`data.import_rows`
960
- (the same gate the CLI uses, scratch copy first), and every file of the updated
961
- tree comes back the way ``init-template`` returns one, so the platform commits a
962
- version from it without knowing which file changed."""
963
- from .data import KIND_SCHEMA, MERGES, MODES, import_rows
964
- from .redaction import parse_mode
965
-
966
- rows = request.get("rows")
967
- mode = request.get("mode", "append")
968
- merge = request.get("merge", "upsert")
969
- if not isinstance(rows, dict) or not all(isinstance(v, list) for v in rows.values()):
970
- fail("input", "Expected rows as {entity: [row, …]}")
971
- if mode not in MODES:
972
- fail("input", f"mode must be one of {', '.join(MODES)}")
973
- if merge not in MERGES:
974
- fail("input", f"merge must be one of {', '.join(MERGES)}")
975
- redact = parse_mode(request.get("redact"))
976
- dry_run = _flag(request, "dry_run", False)
977
- with tempfile.TemporaryDirectory(prefix="gateway-import-") as directory:
978
- root = Path(directory) / "bundle"
979
- root.mkdir()
980
- _write_files(root, request.get("files"))
981
- world = world_root(root)
982
- result = import_rows(world, rows, mode=mode, merge=merge, redact=redact, dry_run=dry_run)
983
- keys = ("kind", "entity_counts", "changed")
984
- report = {"status": "ok", **{key: result[key] for key in keys}}
985
- if dry_run:
986
- return {**report, **{key: result[key] for key in ("proof", "deferred") if key in result}}
987
- if result["kind"] == KIND_SCHEMA and snapshots.default_snapshot(world).is_file():
988
- # The journals grew; a snapshot built before this import would be
989
- # served over them. Rebuild so the tree that travels back is one thing.
990
- from .compiler import build
991
- from .snapshot import build_all
992
-
993
- build_all(world, build(world)) # build(): the contract, no snapshot audit
994
- if world != root and result["kind"] == KIND_SCHEMA:
995
- # The bundle's base seed is derived from the world's data; older runners post it.
996
- from .api import write_base_seed
997
-
998
- write_base_seed(world, root)
999
- return {**report, "files": _tree(root)}
1000
-
1001
-
1002
878
  def _ingest_calls(request: dict) -> dict:
1003
879
  """``ingest-calls``: records into a world the platform holds as files — the twin
1004
880
  of ``import-data`` for the record contract of :mod:`.ingest`. The tree arrives
@@ -1252,7 +1128,7 @@ def _api_bundle(request: dict) -> dict:
1252
1128
  what `gateway worlds schema api-bundle` writes locally, for a CLI with no Python."""
1253
1129
  from .api import build_api_bundle
1254
1130
 
1255
- files = _template_files(request)
1131
+ files = request_files(request)
1256
1132
  with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
1257
1133
  root = Path(directory) / "world"
1258
1134
  root.mkdir()
@@ -1364,11 +1240,29 @@ def _run_tests(request: dict) -> dict:
1364
1240
  return {"status": "ok", "report": report}
1365
1241
 
1366
1242
 
1243
+ def _prepared_tree(request: dict) -> Path:
1244
+ """The tree the platform laid under a ``workdir`` it created — ``tree``, a
1245
+ relative path under it, beside any snapshot it checked out — opened in place.
1246
+ Every path is validated against the workdir (:func:`_under`); the workdir is
1247
+ the caller's to remove."""
1248
+ if request.get("files") is not None or request.get("world") is not None:
1249
+ fail("input", "a request opens its world through one of workdir, files or world")
1250
+ workdir = _workdir(request)
1251
+ tree = _under(workdir, request.get("tree"), "tree")
1252
+ if not tree.is_dir():
1253
+ fail("input", "tree must be a directory under workdir")
1254
+ return world_root(tree)
1255
+
1256
+
1367
1257
  def _world_for(request: dict, staging: Path) -> Path:
1368
1258
  """The world a run-style command works on: ``files`` laid into ``staging``
1369
- (what the platform sends), or — where this runtime may run authored code —
1370
- ``world``, an absolute directory on this machine, opened in place so its
1371
- ``[dependencies]`` paths resolve and nothing is copied."""
1259
+ (what the platform sends for a small tree), the tree under a ``workdir`` the
1260
+ platform prepared (:func:`_prepared_tree`, for a large one), or — where this
1261
+ runtime may run authored code — ``world``, an absolute directory on this
1262
+ machine, opened in place so its ``[dependencies]`` paths resolve and nothing
1263
+ is copied."""
1264
+ if request.get("workdir") is not None:
1265
+ return _prepared_tree(request)
1372
1266
  world = request.get("world")
1373
1267
  if world is None:
1374
1268
  root = staging / "world"
@@ -1492,109 +1386,12 @@ def _dry_run(request: dict) -> dict:
1492
1386
  return answer
1493
1387
 
1494
1388
 
1495
- def _counts(request: dict) -> dict:
1496
- """``counts``: what the world at these files seeds with. A kind whose seed lives
1497
- elsewhere (a harbor tree, an unknown layout) answers with the reason instead of
1498
- a rejection, so the caller can show it."""
1499
- from .data import entity_counts, entity_rows, world_kind
1500
-
1501
- entity = request.get("entity")
1502
- if entity is not None and not isinstance(entity, str):
1503
- fail("input", "entity must be a string")
1504
- with tempfile.TemporaryDirectory(prefix="gateway-counts-") as directory:
1505
- root = Path(directory) / "world"
1506
- root.mkdir()
1507
- _write_files(root, request.get("files"))
1508
- root = world_root(root)
1509
- kind = world_kind(root)
1510
- try:
1511
- counts = entity_counts(root)
1512
- except ContractError as error:
1513
- if error.detail.get("code") != "kind":
1514
- raise
1515
- return {
1516
- "status": "ok",
1517
- "kind": kind,
1518
- "entity_counts": None,
1519
- "reason": error.detail["message"],
1520
- }
1521
- report = {"status": "ok", "kind": kind, "entity_counts": counts}
1522
- if entity is not None:
1523
- report["rows"] = entity_rows(root, entity)
1524
- return report
1525
-
1526
-
1527
- def _query(request: dict) -> dict:
1528
- """``query``: the rows of one entity of the world at these files that match
1529
- ``where`` (a field to a value or a list of values), a page at a time (``limit``,
1530
- ``offset``). With ``resolve``, each value is plaintext and is looked up in the
1531
- forms the world's ``[redact]`` policy stores (:func:`data.entity_query`). A tree
1532
- that carries its compiled ``db/state.sqlite`` (a checked-out snapshot) answers
1533
- from the snapshot, as ``counts`` does; otherwise from its data files. With
1534
- ``across`` (local runtime only: ``world`` is a directory here, so its
1535
- ``[dependencies]`` paths resolve) the answer is :func:`data.entity_query_across`'s
1536
- groups, one per linked world a mapping carries a ``where`` field into."""
1537
- from .data import entity_query, entity_query_across, world_kind
1538
-
1539
- entity = request.get("entity")
1540
- if not isinstance(entity, str) or not entity:
1541
- fail("input", "entity must be a non-empty string")
1542
- where = request.get("where", {})
1543
- if not isinstance(where, dict):
1544
- fail("input", "where must be an object mapping a field to a value or a list of values")
1545
- resolve = request.get("resolve", False)
1546
- if not isinstance(resolve, bool):
1547
- fail("input", "resolve must be a boolean")
1548
- # The platform's cross-world query sends the mapping's transform chain per field
1549
- # under this key (worldDataQueryService.ts BRIDGE_WHERE_TRANSFORM_KEY).
1550
- where_transform = request.get("where_transform")
1551
- if where_transform is not None and not isinstance(where_transform, dict):
1552
- fail("input", "where_transform must be an object mapping a field to a list of transform names")
1553
- across = request.get("across", False)
1554
- compare = request.get("compare", False)
1555
- if not isinstance(across, bool) or not isinstance(compare, bool):
1556
- fail("input", "across and compare must be booleans")
1557
- if compare and not across:
1558
- fail("input", "compare needs across")
1559
- scenario = request.get("scenario")
1560
- if scenario is not None and (not isinstance(scenario, str) or not scenario):
1561
- fail("input", "scenario must be a non-empty string", "/scenario")
1562
- with tempfile.TemporaryDirectory(prefix="gateway-query-") as directory:
1563
- root = _world_for(request, Path(directory))
1564
- if across:
1565
- if request.get("world") is None:
1566
- fail(
1567
- "input", "across reads the linked worlds at their paths, so it takes world (a directory), not files"
1568
- )
1569
- report = entity_query_across(
1570
- root,
1571
- entity,
1572
- where,
1573
- limit=request.get("limit", 100),
1574
- offset=request.get("offset", 0),
1575
- resolve=resolve,
1576
- compare=compare,
1577
- scenario=scenario,
1578
- )
1579
- return {"status": "ok", "kind": world_kind(root), **report}
1580
- report = entity_query(
1581
- root,
1582
- entity,
1583
- where,
1584
- limit=request.get("limit", 100),
1585
- offset=request.get("offset", 0),
1586
- resolve=resolve,
1587
- where_transform=where_transform,
1588
- scenario=scenario,
1589
- )
1590
- return {"status": "ok", "kind": world_kind(root), **report}
1591
-
1592
-
1593
1389
  def _templates(request: dict) -> dict:
1594
1390
  """``templates``: every installed connector template, read from its ``world.json``
1595
1391
  (name and description when the document carries them) and whether it declares a
1596
1392
  vendor connection. Never runs a template's Python."""
1597
1393
  from .scaffold import TEMPLATES, list_templates
1394
+ from .ui import ui_kind
1598
1395
 
1599
1396
  templates = []
1600
1397
  for slug in list_templates():
@@ -1610,6 +1407,7 @@ def _templates(request: dict) -> dict:
1610
1407
  "entities": sorted(gateway.get("entities") or {}),
1611
1408
  "tools": sorted(gateway.get("tools") or {}),
1612
1409
  "hasConnection": (TEMPLATES / slug / "connector.toml").is_file(),
1410
+ "ui": ui_kind(TEMPLATES / slug),
1613
1411
  }
1614
1412
  )
1615
1413
  return {"status": "ok", "templates": templates}
@@ -1630,7 +1428,11 @@ def execute(request: dict) -> dict:
1630
1428
  if request["command"] == "capture":
1631
1429
  return _capture(request)
1632
1430
  if request["command"] == "import-data":
1633
- return _import_data(request)
1431
+ return platform_import.import_data(request)
1432
+ if request["command"] == "import-files":
1433
+ return platform_import.import_files(request, _world_for)
1434
+ if request["command"] == "chunk-files":
1435
+ return platform_import.chunk_files(request, _world_for)
1634
1436
  if request["command"] == "ingest":
1635
1437
  return _ingest(request)
1636
1438
  if request["command"] == "ingest-calls":
@@ -1646,15 +1448,15 @@ def execute(request: dict) -> dict:
1646
1448
  if request["command"] == "dry-run":
1647
1449
  return _dry_run(request)
1648
1450
  if request["command"] == "counts":
1649
- return _counts(request)
1451
+ return platform_query.counts(request, _world_for)
1650
1452
  if request["command"] == "query":
1651
- return _query(request)
1453
+ return platform_query.query(request, _world_for)
1652
1454
  if request["command"] == "templates":
1653
1455
  return _templates(request)
1654
1456
  if request["command"] == "api-bundle":
1655
1457
  return _api_bundle(request)
1656
1458
  if request["command"] == "template-summary":
1657
- return _template_summary(_template_files(request))
1459
+ return _template_summary(request_files(request))
1658
1460
  if request["command"] == "describe":
1659
1461
  return _describe(request)
1660
1462
  extra: dict = {}
@@ -1723,7 +1525,14 @@ def execute(request: dict) -> dict:
1723
1525
  name = request.get("name")
1724
1526
  if not isinstance(connector, str) or (name is not None and not isinstance(name, str)):
1725
1527
  fail("input", "Expected the connector slug and an optional name")
1726
- init_bundle_from_files(root, _template_files(request), connector=connector, name=name)
1528
+ init_bundle_from_files(root, request_files(request), connector=connector, name=name)
1529
+ if request["command"] == "init-openapi":
1530
+ from .openapi_import import OpenApiImport
1531
+
1532
+ planned = OpenApiImport.execute(request, root)
1533
+ if not planned["written"]:
1534
+ return planned
1535
+ extra.update(planned)
1727
1536
  if request["command"] == "build-slack":
1728
1537
  from .slack import build_slack_bundle
1729
1538
 
@@ -1737,13 +1546,15 @@ def execute(request: dict) -> dict:
1737
1546
  **extra,
1738
1547
  }
1739
1548
  if request["command"] not in ("validate", "validate-archive"):
1549
+ paths = [path for path in sorted(root.rglob("*")) if path.is_file()]
1550
+ if len(paths) > MAX_FILES:
1551
+ fail("size", f"The world tree has {len(paths)} files, over the {MAX_FILES} limit")
1740
1552
  report["files"] = [
1741
1553
  {
1742
1554
  "path": str(path.relative_to(root)),
1743
1555
  "content": base64.b64encode(path.read_bytes()).decode("ascii"),
1744
1556
  }
1745
- for path in sorted(root.rglob("*"))
1746
- if path.is_file()
1557
+ for path in paths
1747
1558
  ]
1748
1559
  return report
1749
1560