@withgateway/sdk 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/cli.js +3121 -1283
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +3185 -1347
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/files/index.js +9 -0
  6. package/dist/files/index.js.map +1 -1
  7. package/dist/files/index.mjs +9 -0
  8. package/dist/files/index.mjs.map +1 -1
  9. package/dist/hooks/index.d.mts +1 -1
  10. package/dist/hooks/index.d.ts +1 -1
  11. package/dist/hooks/index.js +280 -8
  12. package/dist/hooks/index.js.map +1 -1
  13. package/dist/hooks/index.mjs +272 -8
  14. package/dist/hooks/index.mjs.map +1 -1
  15. package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
  16. package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
  17. package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
  18. package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
  19. package/dist/index.d.mts +4 -4
  20. package/dist/index.d.ts +4 -4
  21. package/dist/index.js +726 -167
  22. package/dist/index.js.map +1 -1
  23. package/dist/index.mjs +718 -167
  24. package/dist/index.mjs.map +1 -1
  25. package/dist/traces/index.js +9 -0
  26. package/dist/traces/index.js.map +1 -1
  27. package/dist/traces/index.mjs +9 -0
  28. package/dist/traces/index.mjs.map +1 -1
  29. package/dist/tracing/index.d.mts +1 -1
  30. package/dist/tracing/index.d.ts +1 -1
  31. package/dist/tracing/index.js +97 -61
  32. package/dist/tracing/index.js.map +1 -1
  33. package/dist/tracing/index.mjs +97 -61
  34. package/dist/tracing/index.mjs.map +1 -1
  35. package/dist/worlds/index.d.mts +154 -4
  36. package/dist/worlds/index.d.ts +154 -4
  37. package/dist/worlds/index.js +348 -97
  38. package/dist/worlds/index.js.map +1 -1
  39. package/dist/worlds/index.mjs +348 -97
  40. package/dist/worlds/index.mjs.map +1 -1
  41. package/package.json +1 -1
  42. package/runtime/gatewaysdk/environment/_bundle.py +1 -1
  43. package/runtime/gatewaysdk/environment/_materialize.py +53 -9
  44. package/runtime/gatewaysdk/environment/_toml.py +13 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +3 -7
  46. package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
  47. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
  48. package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
  49. package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
  51. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
  52. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
  53. package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
  54. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
  55. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  56. package/runtime/gatewaysdk/environment/schema/platform.py +16 -4
  57. package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
  58. package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
  59. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
  60. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +61 -2
  61. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
  62. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
  63. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
  64. package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
  65. package/runtime/gatewaysdk/environment/schema/tools_world.py +1 -5
  66. package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
  67. package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
  68. package/runtime/gatewaysdk/environment/world.py +120 -4
  69. package/skills/benchmarks-getting-started/SKILL.md +67 -9
  70. package/skills/connector-schema-authoring/connector-worlds.md +8 -1
  71. package/skills/connector-schema-authoring/world-schema.md +1 -1
  72. package/skills/hooks-getting-started/SKILL.md +61 -2
  73. package/skills/world-data-ingestion/SKILL.md +4 -1
  74. package/skills/worlds-getting-started/SKILL.md +11 -11
  75. package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@withgateway/sdk",
3
- "version": "0.2.5",
3
+ "version": "0.2.6",
4
4
  "description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
5
5
  "main": "./dist/index.js",
6
6
  "module": "./dist/index.mjs",
@@ -17,12 +17,12 @@ import functools
17
17
  import importlib.util
18
18
  import inspect
19
19
  import sys
20
- import tomllib
21
20
  import uuid
22
21
  from dataclasses import dataclass, field
23
22
  from pathlib import Path
24
23
  from typing import Any, Callable, Dict, List, Optional
25
24
 
25
+ from ._toml import tomllib
26
26
  from . import runtime
27
27
  from .errors import EnvironmentNotReady, WorldStartError
28
28
  from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
@@ -11,7 +11,9 @@ from tempfile import TemporaryDirectory
11
11
  from ._bundle import load_handlers, parse_manifest
12
12
  from .errors import WorldStartError
13
13
  from .schema import SQLiteStore, bundle_mode, check_bundle
14
- from .schema.validation import read_json
14
+ from .schema.batch import shape_refusal
15
+ from .schema.compiler import Contract
16
+ from .schema.validation import ContractError, fail, read_json, validate
15
17
 
16
18
 
17
19
  def materialize_template(bundle: Path, destination: Path, *, require_sql: bool = False) -> None:
@@ -19,8 +21,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
19
21
  if bundle_mode(bundle) is not None:
20
22
  contract = check_bundle(bundle)
21
23
  if destination.exists():
22
- from .schema import ContractError
23
-
24
24
  raise ContractError("state_exists", "Refusing to overwrite an existing template")
25
25
  from .schema.snapshot import current_default, default_snapshot, pin_path
26
26
 
@@ -33,8 +33,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
33
33
  if pin_path(bundle).is_file():
34
34
  # A pinned tree's data is the hosted snapshot; its data files cannot
35
35
  # stand in for it, so a missing or stale snapshot is refused, not rebuilt.
36
- from .schema import ContractError
37
-
38
36
  raise ContractError(
39
37
  "snapshot_pinned",
40
38
  "db/snapshot.json pins a hosted snapshot that is not in the tree: run the world on "
@@ -43,6 +41,7 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
43
41
  schema_path="/db/state.sqlite",
44
42
  )
45
43
  initial = read_json(bundle / "data/initial.json")
44
+ refuse_initial_rows(contract, initial)
46
45
  try:
47
46
  with SQLiteStore.create(destination, contract) as store:
48
47
  store.apply_batch(initial)
@@ -50,14 +49,10 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
50
49
  # Coverage never prunes: every change is an explicit store mutation.
51
50
  batches = bundle / "data/batches"
52
51
  if batches.is_symlink():
53
- from .schema import ContractError
54
-
55
52
  raise ContractError("path", "Refresh batches cannot be symlinks")
56
53
  if batches.exists():
57
54
  for batch in sorted(batches.iterdir()):
58
55
  if batch.is_symlink() or not batch.is_file() or batch.suffix != ".json":
59
- from .schema import ContractError
60
-
61
56
  raise ContractError("path", "Refresh journals must be regular JSON files")
62
57
  store.apply_batch(read_json(batch))
63
58
  except BaseException:
@@ -80,6 +75,55 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
80
75
  conn.close()
81
76
 
82
77
 
78
+ def refuse_initial_rows(contract: Contract, initial: object) -> None:
79
+ """Every inserted row's shape in ``data/initial.json`` against the contract,
80
+ all of them, before the store applies any: one ``ContractError`` with the
81
+ first refusal's own code, its entity, mutation index, primary key and
82
+ pointer, carrying the full ``refusals`` list — the shape the data-import
83
+ lane reports, so a hosted gate names the row and not just the rule. Rows
84
+ the store would refuse for other reasons (references, duplicate keys)
85
+ still surface from ``apply_batch`` with the mutation's index and key."""
86
+ if not isinstance(initial, dict) or not isinstance(initial.get("mutations"), list):
87
+ return
88
+ inserts = [
89
+ (index, mutation)
90
+ for index, mutation in enumerate(initial["mutations"])
91
+ if isinstance(mutation, dict) and mutation.get("op") == "insert" and isinstance(mutation.get("entity"), str)
92
+ ]
93
+ refusals = [
94
+ refused
95
+ for index, mutation in inserts
96
+ if (refused := shape_refusal(contract, 0, index, mutation["entity"], mutation.get("value"))) is not None
97
+ ]
98
+ if not refusals:
99
+ return
100
+ first = refusals[0]
101
+ summary = f"{len(refusals)} of {len(inserts)} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
102
+ context = {
103
+ "file": "data/initial.json",
104
+ "entity": first["entity"],
105
+ "index": first["index"],
106
+ "key": first["primaryKey"],
107
+ "field": first["field"],
108
+ "refused": len(refusals),
109
+ "rows": len(inserts),
110
+ "refusals": refusals,
111
+ }
112
+ row = initial["mutations"][first["index"]].get("value")
113
+ spec = contract.entities.get(first["entity"])
114
+ if spec is None:
115
+ fail("entity", summary + first["detail"], first["pointer"], **context)
116
+ if not isinstance(row, dict):
117
+ fail("rows", summary + first["detail"], first["pointer"], **context)
118
+ if any(name not in row for name in spec["primary_key"]):
119
+ fail("primary_key", summary + first["detail"], first["pointer"], **context)
120
+ try:
121
+ validate(spec["schema"], row, location=f"{spec['location']}/schema")
122
+ except ContractError as error:
123
+ raise error.contextualize(message=summary + error.detail["message"], **context)
124
+ fail("rows", summary + first["detail"], first["pointer"], **context)
125
+
126
+
83
127
  def validate_schema_materialization(bundle: Path) -> None:
84
128
  """Validate compiled artifacts and initial state without retaining a database."""
85
129
  bundle = Path(bundle)
@@ -0,0 +1,13 @@
1
+ """The one TOML reader for every Python the SDK supports.
2
+
3
+ 3.11+ ships ``tomllib``; 3.10 gets ``tomli`` (declared in pyproject as
4
+ ``tomli; python_version < '3.11'``). Import ``tomllib`` from here, never the
5
+ stdlib module directly, so the fallback lives in one place.
6
+ """
7
+
8
+ try:
9
+ import tomllib
10
+ except ModuleNotFoundError: # pragma: no cover - python 3.10
11
+ import tomli as tomllib # type: ignore[no-redef]
12
+
13
+ __all__ = ["tomllib"]
@@ -95,7 +95,6 @@ import sys
95
95
  import tempfile
96
96
  import threading
97
97
  import time
98
- import tomllib
99
98
  import urllib.parse
100
99
  import uuid
101
100
  from dataclasses import dataclass, field
@@ -104,7 +103,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
104
103
  from pathlib import Path
105
104
  from typing import Any, Callable
106
105
 
106
+ from .._toml import tomllib
107
107
  from . import snapshot as snapshots
108
+ from .bundle_runtime import stage_runtime_package
108
109
  from .compiler import check_bundle, compile_bundle
109
110
  from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
110
111
  from .grading import keyed_entities
@@ -1973,12 +1974,7 @@ def build_api_bundle(world: str | Path, out: str | Path) -> Path:
1973
1974
  f"The {spec.slug} world serving the vendor's API, seeded with its captured state. Point the provider client here.\n"
1974
1975
  )
1975
1976
  server = out_dir / "server"
1976
- package = server / "gatewaysdk" / "environment" / "schema"
1977
- package.mkdir(parents=True)
1978
- (server / "gatewaysdk" / "__init__.py").write_text("")
1979
- (server / "gatewaysdk" / "environment" / "__init__.py").write_text("")
1980
- for module in sorted(Path(__file__).parent.glob("*.py")):
1981
- shutil.copy(module, package / module.name)
1977
+ stage_runtime_package(server)
1982
1978
  shutil.copytree(root, server / "world", ignore=shutil.ignore_patterns("captures", "__pycache__", ".git"))
1983
1979
  (server / "main.py").write_text(
1984
1980
  "\n".join(
@@ -355,8 +355,11 @@ def chunk_rows(
355
355
  chunk_bytes: int = CHUNK_BYTES,
356
356
  chunk_rows: int | None = None,
357
357
  entity: str | None = None,
358
+ first_ordinal: int = 0,
358
359
  ) -> Iterator[Chunk]:
359
- """Cut a row stream into gzipped JSONL chunk files under ``workdir``.
360
+ """Cut a row stream into gzipped JSONL chunk files under ``workdir``, numbering
361
+ them from ``first_ordinal`` (the ordinals below it belong to chunks the platform
362
+ staged from the project drive).
360
363
 
361
364
  Lines are tagged ``{"entity", "row"}`` unless ``entity`` names the one entity
362
365
  every row belongs to, in which case they are bare rows. A chunk closes when the
@@ -367,8 +370,10 @@ def chunk_rows(
367
370
  workdir.mkdir(parents=True, exist_ok=True)
368
371
  if chunk_bytes < 1 or (chunk_rows is not None and chunk_rows < 1):
369
372
  fail("chunk", "chunk bounds must be positive", "/chunk")
373
+ if first_ordinal < 0:
374
+ fail("chunk", "the first ordinal must not be negative", "/chunk")
370
375
  writer: _ChunkWriter | None = None
371
- ordinal = 0
376
+ ordinal = first_ordinal
372
377
  seen = 0
373
378
  for name, row in rows:
374
379
  if entity is not None and name != entity:
@@ -391,17 +396,18 @@ def chunk_rows(
391
396
 
392
397
 
393
398
  def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tuple[str, dict]]:
394
- """Stream ``(entity, row)`` from one chunk file (gzip over JSONL; a plain file
395
- is read as is), tagged lines or bare rows of the declared entity."""
399
+ """Stream ``(entity, row)`` from one chunk file: gzip (sniffed, whatever the name)
400
+ or plain, holding any rows-file shape — the tagged or bare JSONL a CLI cut, or a
401
+ ``{entity: [rows]}`` / bare-array document a project-drive file was staged as."""
396
402
  path = Path(path)
397
403
  with open(path, "rb") as probe:
398
404
  compressed = probe.read(2) == b"\x1f\x8b"
399
405
  opener = gzip.open if compressed else open
400
406
  with opener(path, "rt", encoding="utf-8") as stream: # type: ignore[call-overload]
401
- reader = _Reader(stream)
402
- first = reader.line()
403
- if first is not None:
404
- yield from _stream_jsonl(reader, first, str(path), entity)
407
+ try:
408
+ yield from iter_file_rows(stream, str(path), entity=entity)
409
+ except UnicodeError:
410
+ fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
405
411
 
406
412
 
407
413
  # ---------------------------------------------------------------- refusals
@@ -669,9 +675,7 @@ def _apply(
669
675
  return applied, None
670
676
 
671
677
 
672
- def validate_chunks(
673
- contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None
674
- ) -> dict:
678
+ def validate_chunks(contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None) -> dict:
675
679
  """``validate-rows``: shape-only, every row of every chunk — after the batch's
676
680
  redaction hook, when it has one, so a plaintext row under ``redact=refuse``
677
681
  is refused here, in the validating phase, before anything is applied."""
@@ -0,0 +1,24 @@
1
+ """The runtime an api bundle carries beside its `server/main.py`: this package, vendored."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import shutil
6
+ from pathlib import Path
7
+
8
+ SCHEMA_PACKAGE = Path(__file__).parent
9
+ ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
10
+ # Modules outside `schema/` that its modules import; copied beside them so `main.py` resolves them.
11
+ SIBLING_MODULES = ("_toml.py",)
12
+
13
+
14
+ def stage_runtime_package(server: Path) -> Path:
15
+ """Vendor `gatewaysdk.environment.schema` (and what it imports) under `server/`; returns the package dir."""
16
+ package = server / "gatewaysdk" / "environment" / "schema"
17
+ package.mkdir(parents=True)
18
+ for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
19
+ (server / package_dir / "__init__.py").write_text("")
20
+ for name in SIBLING_MODULES:
21
+ shutil.copy(ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name)
22
+ for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
23
+ shutil.copy(module, package / module.name)
24
+ return package
@@ -8,11 +8,7 @@ from dataclasses import dataclass
8
8
  from pathlib import Path
9
9
  from typing import Any
10
10
 
11
- try:
12
- import tomllib
13
- except ModuleNotFoundError:
14
- import tomli as tomllib
15
-
11
+ from .._toml import tomllib
16
12
  from .validation import (
17
13
  LANGUAGE,
18
14
  ContractError,
@@ -51,6 +47,7 @@ FIELD_RESERVED = ("sqlite_", "gateway_", "benchmark_")
51
47
  #: An entity may not take them.
52
48
  RESERVED_STATE_KEYS = frozenset({"snapshot", "calls", "report"})
53
49
 
50
+
54
51
  def identifier(value: Any, location: str) -> str:
55
52
  if (
56
53
  not isinstance(value, str)
@@ -632,6 +629,39 @@ def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
632
629
  return contract
633
630
 
634
631
 
632
+ STALE_JOURNAL = (
633
+ "Batch targets a different domain identity/version/hash; a journal that holds "
634
+ "state is not migrated to an edited schema — re-import its rows against the new "
635
+ "contract (or restore the schema it was written against)"
636
+ )
637
+
638
+
639
+ def _journals_off_target(bundle: Path, contract: Contract) -> list[tuple[Path, str, dict]]:
640
+ """Every journal whose target is not this contract's: ``(path, location, journal)``."""
641
+ from .validation import journal_paths, read_json
642
+
643
+ off = []
644
+ for path in journal_paths(bundle):
645
+ journal = read_json(path)
646
+ location = str(path.relative_to(bundle))
647
+ if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
648
+ fail("batch", "Journal must be a batch object with a target", location)
649
+ if journal["target"] != contract.target:
650
+ off.append((path, location, journal))
651
+ return off
652
+
653
+
654
+ def stale_journals(bundle: str | Path, contract: Contract) -> list[str]:
655
+ """The journals ``compile`` will refuse: rows written against another
656
+ contract. Read-only; what ``check`` names instead of \"run compile\" when
657
+ compile is the step that cannot succeed."""
658
+ return [
659
+ location
660
+ for _path, location, journal in _journals_off_target(Path(bundle), contract)
661
+ if journal.get("mutations") != []
662
+ ]
663
+
664
+
635
665
  def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]:
636
666
  """Re-stamp every journal that holds no state onto the compiled contract's
637
667
  target, and refuse one that holds rows.
@@ -642,25 +672,11 @@ def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]
642
672
  migrate in an empty journal, whatever domain it named, so it follows the
643
673
  contract. A journal with rows is never migrated silently; the message names
644
674
  the fix. Returns the bundle-relative paths rewritten."""
645
- from .validation import journal_paths, read_json
646
-
647
675
  bundle = Path(bundle)
648
676
  retargeted = []
649
- for path in journal_paths(bundle):
650
- journal = read_json(path)
651
- location = str(path.relative_to(bundle))
652
- if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
653
- fail("batch", "Journal must be a batch object with a target", location)
654
- if journal["target"] == contract.target:
655
- continue
677
+ for path, location, journal in _journals_off_target(bundle, contract):
656
678
  if journal.get("mutations") != []:
657
- fail(
658
- "batch_target",
659
- "Batch targets a different domain identity/version/hash; a journal that holds "
660
- "state is not migrated to an edited schema — re-import its rows against the new "
661
- "contract (or restore the schema it was written against)",
662
- location,
663
- )
679
+ fail("batch_target", STALE_JOURNAL, location)
664
680
  journal["target"] = contract.target
665
681
  path.write_bytes(encoded(journal))
666
682
  retargeted.append(location)
@@ -690,18 +706,34 @@ def check_bundle(bundle: str | Path) -> Contract:
690
706
  except OSError:
691
707
  matches = False
692
708
  if not matches:
693
- fail(
694
- "lock_mismatch",
695
- "Missing, edited or incompatible contract artifact; the tree changed "
696
- "since its last compile, so run `gateway worlds schema compile <dir>` "
697
- "(state migration is not automatic)",
698
- path,
699
- )
709
+ fail("lock_mismatch", _stale_artifact_remedy(bundle, contract), path)
700
710
  from .snapshot import check_all
701
711
 
702
712
  check_all(bundle, contract)
703
713
  return contract
704
714
 
705
715
 
716
+ def _stale_artifact_remedy(bundle: Path, contract: Contract) -> str:
717
+ """What to do about a stale artifact. \"Run compile\" only when compile can
718
+ succeed; when a journal holds rows written against the previous contract,
719
+ compile refuses too, so name that step and its fix here (a check that sent
720
+ the author to a compile that failed the same way cost a dogfood session)."""
721
+ stale = stale_journals(bundle, contract)
722
+ if not stale:
723
+ return (
724
+ "Missing, edited or incompatible contract artifact; the tree changed "
725
+ "since its last compile, so run `gateway worlds schema compile <dir>` "
726
+ "(state migration is not automatic)"
727
+ )
728
+ return (
729
+ "Missing, edited or incompatible contract artifact; the schema changed since "
730
+ f"its last compile and {', '.join(stale)} holds rows written against the previous "
731
+ "contract, so `gateway worlds schema compile <dir>` will refuse too (batch_target). "
732
+ "Either restore the schema those rows were written against, or move the journal "
733
+ "aside, compile, and re-import the rows against the new contract "
734
+ "(`gateway worlds data import <dir> <rows.json>`)"
735
+ )
736
+
737
+
706
738
  def describe_bundle(bundle: str | Path) -> str:
707
739
  return build(Path(bundle)).artifacts["schema/README.md"].decode()
@@ -29,7 +29,6 @@ from __future__ import annotations
29
29
  import base64
30
30
  import json
31
31
  import threading
32
- import tomllib
33
32
  import urllib.error
34
33
  import urllib.parse
35
34
  import urllib.request
@@ -37,6 +36,7 @@ from dataclasses import dataclass, field
37
36
  from pathlib import Path
38
37
  from typing import Any
39
38
 
39
+ from .._toml import tomllib
40
40
  from .api import WorldState, load_api_settings, serve
41
41
  from .connector import SPEC, Operation, load_spec
42
42
  from .validation import fail
@@ -66,7 +66,6 @@ import re
66
66
  import shutil
67
67
  import tempfile
68
68
  import time
69
- import tomllib
70
69
  import urllib.error
71
70
  import urllib.parse
72
71
  import urllib.request
@@ -75,6 +74,7 @@ from datetime import datetime, timezone
75
74
  from pathlib import Path
76
75
  from typing import Any, Callable
77
76
 
77
+ from .._toml import tomllib
78
78
  from .compiler import check_bundle, compile_bundle
79
79
  from .store import SQLiteStore
80
80
  from .transforms import apply_transforms
@@ -0,0 +1,25 @@
1
+ # Merge `google_drive` tool results onto this world's entities. Merge answers each
2
+ # tool with one JSON document; `list_files` carries the files under `files`, in
3
+ # snake_case (the Drive contract is camelCase, hence the renames).
4
+
5
+ [ingest]
6
+ on_conflict = "latest"
7
+ unmatched = "refuse"
8
+
9
+ [[sources]]
10
+ name = "list_files"
11
+ select = "result.files[*]"
12
+ entity = "drive_files"
13
+ at = "modified_time"
14
+
15
+ [sources.fields]
16
+ id = "id"
17
+ name = "name"
18
+ mimeType = "mime_type"
19
+ starred = "starred"
20
+ trashed = "trashed"
21
+ shared = "shared"
22
+ size = "size"
23
+ webViewLink = "web_view_link"
24
+ createdTime = "created_time"
25
+ modifiedTime = "modified_time"
@@ -0,0 +1,17 @@
1
+ # How the platform fills a world on this contract from a live Merge connection
2
+ # (`gateway worlds create --template google-drive --mode live --connection <id>`).
3
+ # Each [[pulls]] is one read tool of Merge's `google_drive` tool pack, called on
4
+ # the connection's registered user; readers/merge.ingest.toml maps what comes back
5
+ # onto the world's entities. Only read tools belong here.
6
+
7
+ [reader]
8
+ transport = "merge"
9
+ provider = "google_drive"
10
+ mapping = "merge.ingest.toml"
11
+
12
+ # Files the connected account can see; trashed ones stay out of the world.
13
+ [[pulls]]
14
+ tool = "list_files"
15
+ arguments = { page_size = 100, q = "trashed = false" }
16
+ page = { cursor = "page_token", next = "next_page_token" }
17
+ max_pages = 20
@@ -31,6 +31,7 @@ from dataclasses import dataclass, field
31
31
  from pathlib import Path
32
32
  from typing import IO, Any, Iterable, Iterator
33
33
 
34
+ from .._toml import tomllib
34
35
  from .batch import _Reader, _stream_array, encode_row
35
36
  from .compiler import Contract, build
36
37
  from .connector import SPEC, Spec, redact
@@ -63,11 +64,6 @@ class Summary:
63
64
 
64
65
  def envelope_key(world: Path) -> str:
65
66
  """``[api].envelope`` of the world's connector.toml; "" when unset or absent."""
66
- try:
67
- import tomllib
68
- except ModuleNotFoundError: # pragma: no cover - python 3.10
69
- import tomli as tomllib
70
-
71
67
  spec = world / SPEC
72
68
  if not spec.is_file():
73
69
  return ""
@@ -35,11 +35,7 @@ from dataclasses import dataclass
35
35
  from http.cookies import SimpleCookie
36
36
  from pathlib import Path
37
37
 
38
- try:
39
- import tomllib
40
- except ModuleNotFoundError: # pragma: no cover - python < 3.11
41
- import tomli as tomllib # type: ignore[no-redef]
42
-
38
+ from .._toml import tomllib
43
39
  from .validation import fail
44
40
 
45
41
  API_BUNDLE = "gateway-api-world.toml"
@@ -51,12 +51,12 @@ from __future__ import annotations
51
51
  import hashlib
52
52
  import json
53
53
  import re
54
- import tomllib
55
54
  from dataclasses import dataclass
56
55
  from datetime import datetime, timezone
57
56
  from pathlib import Path
58
57
  from typing import Any, Iterable
59
58
 
59
+ from .._toml import tomllib
60
60
  from .transforms import apply_transforms, parse_transform
61
61
  from .validation import ContractError, fail
62
62
 
@@ -326,7 +326,7 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
326
326
  "domain": contract.target,
327
327
  "package": contract.metadata["package"],
328
328
  "initial_state": initial_state,
329
- "handler_execution": _handler_execution(bundle, execute),
329
+ "handler_execution": _handler_execution(bundle, execute, declared),
330
330
  "entity_counts": counts,
331
331
  "tools": sorted(declared),
332
332
  "verifiers": verifiers,
@@ -527,10 +527,14 @@ def may_execute() -> bool:
527
527
  return os.environ.get(EXECUTE_ENV) == "1"
528
528
 
529
529
 
530
- def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
530
+ def _handler_execution(bundle: Path, execute: bool, declared: set[str]) -> dict[str, Any]:
531
531
  """Whether the handlers were run: imported, every declared tool bound, the
532
532
  ``[dependencies]`` opened from their paths — one session of the world,
533
- opened and closed. Only where authored code may run."""
533
+ opened and closed. Only where authored code may run. ``tools`` is the
534
+ declared handlers the session bound (the report's top-level ``tools``);
535
+ ``served`` every name ``worlds call`` answers there — handlers, the connector
536
+ routes as operations, a dependency's ``alias.tool`` — so the two lists differ
537
+ by design and say what each counts."""
534
538
  if not execute:
535
539
  return {
536
540
  "status": "not_run",
@@ -546,7 +550,15 @@ def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
546
550
  from .local import open_world
547
551
 
548
552
  with open_world(bundle, http=False) as session:
549
- return {"status": "ok", "tools": session.tools}
553
+ served = list(session.tools)
554
+ unbound = sorted(declared - set(served))
555
+ if unbound:
556
+ fail(
557
+ "handler",
558
+ "Declared in schema/world.json (tools) but not bound when the session opened: " + ", ".join(unbound),
559
+ "/tools",
560
+ )
561
+ return {"status": "ok", "tools": sorted(declared), "served": served}
550
562
 
551
563
 
552
564
  def world_root(root: Path) -> Path:
@@ -31,11 +31,11 @@ from __future__ import annotations
31
31
 
32
32
  import json
33
33
  import re
34
- import tomllib
35
34
  from dataclasses import dataclass, field
36
35
  from pathlib import Path
37
36
  from typing import Any, Callable
38
37
 
38
+ from .._toml import tomllib
39
39
  from .validation import ContractError, fail
40
40
 
41
41
  REDACT_MODES = ("off", "apply", "refuse")
@@ -81,6 +81,11 @@ TEMPLATE_CODE = {"handlers.py", "api/handlers.py"}
81
81
  #: Python (``api/handlers.py``), its served scenarios (``scenarios/…``) and the
82
82
  #: tests it ships for its tools (``tests/…``, run by ``gateway worlds test``).
83
83
  TEMPLATE_DIRECTORIES = {"api", "scenarios", "tests", "ui"}
84
+ #: Template subdirectories that stay with the connector package and never enter a
85
+ #: world: authoring evidence, fidelity reports, and ``readers/`` — how the platform
86
+ #: fills a world on this contract from a live connection (``readers/merge.toml``
87
+ #: names the tools to pull, ``readers/merge.ingest.toml`` maps their payloads).
88
+ CONNECTOR_ONLY_DIRECTORIES = {"evidence", "reports", "readers"}
84
89
 
85
90
 
86
91
  def _template_directory_files(directory: Path) -> list[Path]:
@@ -149,8 +154,8 @@ def init_bundle_from_files(
149
154
  target = destination / "connector.toml"
150
155
  elif path.startswith(("api/", "scenarios/", "tests/", "ui/")):
151
156
  target = destination / path
152
- elif path.startswith("evidence/") or path.startswith("reports/"):
153
- continue # authoring evidence stays with the connector, not the world
157
+ elif path.split("/", 1)[0] in CONNECTOR_ONLY_DIRECTORIES:
158
+ continue # evidence, reports and reader plans stay with the connector, not the world
154
159
  elif (
155
160
  Path(path).suffix in SUPPORTING_SUFFIXES and Path(path).name not in GENERATED_NAMES and "/" not in path
156
161
  ):
@@ -231,6 +236,8 @@ def init_bundle(
231
236
  if not (path.name == "tests" and connector == CUSTOM and not with_example_tool):
232
237
  code_files.extend(_template_directory_files(path))
233
238
  continue
239
+ if path.name in CONNECTOR_ONLY_DIRECTORIES and path.is_dir() and not path.is_symlink():
240
+ continue
234
241
  _resource(path)
235
242
  if path.name in {"world.json", "handlers.py"}:
236
243
  continue