@withgateway/sdk 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/cli.js +4304 -2381
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +4266 -2343
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/files/index.js +56 -31
  6. package/dist/files/index.js.map +1 -1
  7. package/dist/files/index.mjs +52 -27
  8. package/dist/files/index.mjs.map +1 -1
  9. package/dist/hooks/index.d.mts +1 -1
  10. package/dist/hooks/index.d.ts +1 -1
  11. package/dist/hooks/index.js +280 -8
  12. package/dist/hooks/index.js.map +1 -1
  13. package/dist/hooks/index.mjs +272 -8
  14. package/dist/hooks/index.mjs.map +1 -1
  15. package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
  16. package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
  17. package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
  18. package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
  19. package/dist/index.d.mts +4 -4
  20. package/dist/index.d.ts +4 -4
  21. package/dist/index.js +1100 -438
  22. package/dist/index.js.map +1 -1
  23. package/dist/index.mjs +1079 -424
  24. package/dist/index.mjs.map +1 -1
  25. package/dist/traces/index.js +45 -20
  26. package/dist/traces/index.js.map +1 -1
  27. package/dist/traces/index.mjs +41 -16
  28. package/dist/traces/index.mjs.map +1 -1
  29. package/dist/tracing/index.d.mts +1 -1
  30. package/dist/tracing/index.d.ts +1 -1
  31. package/dist/tracing/index.js +98 -61
  32. package/dist/tracing/index.js.map +1 -1
  33. package/dist/tracing/index.mjs +98 -61
  34. package/dist/tracing/index.mjs.map +1 -1
  35. package/dist/worlds/index.d.mts +179 -25
  36. package/dist/worlds/index.d.ts +179 -25
  37. package/dist/worlds/index.js +639 -284
  38. package/dist/worlds/index.js.map +1 -1
  39. package/dist/worlds/index.mjs +631 -276
  40. package/dist/worlds/index.mjs.map +1 -1
  41. package/package.json +2 -1
  42. package/runtime/gatewaysdk/environment/_bundle.py +15 -25
  43. package/runtime/gatewaysdk/environment/_materialize.py +53 -9
  44. package/runtime/gatewaysdk/environment/_toml.py +13 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +40 -25
  46. package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
  47. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
  48. package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
  49. package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
  51. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
  52. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
  53. package/runtime/gatewaysdk/environment/schema/data.py +21 -19
  54. package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
  55. package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
  56. package/runtime/gatewaysdk/environment/schema/host.py +156 -546
  57. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
  58. package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
  59. package/runtime/gatewaysdk/environment/schema/local.py +32 -126
  60. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  61. package/runtime/gatewaysdk/environment/schema/platform.py +30 -26
  62. package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
  63. package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
  64. package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
  65. package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
  66. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
  68. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +67 -6
  69. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
  70. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
  71. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
  72. package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
  73. package/runtime/gatewaysdk/environment/schema/tools_world.py +71 -23
  74. package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
  75. package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
  76. package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
  77. package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
  78. package/runtime/gatewaysdk/environment/world.py +507 -0
  79. package/skills/benchmarks-getting-started/SKILL.md +67 -9
  80. package/skills/connector-schema-authoring/connector-worlds.md +8 -1
  81. package/skills/connector-schema-authoring/world-schema.md +1 -1
  82. package/skills/hooks-getting-started/SKILL.md +67 -6
  83. package/skills/world-data-ingestion/SKILL.md +4 -1
  84. package/skills/worlds-getting-started/SKILL.md +11 -11
  85. package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@withgateway/sdk",
3
- "version": "0.2.4",
3
+ "version": "0.2.6",
4
4
  "description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
5
5
  "main": "./dist/index.js",
6
6
  "module": "./dist/index.mjs",
@@ -104,6 +104,7 @@
104
104
  "typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
105
105
  "test": "vitest",
106
106
  "test:run": "vitest run",
107
+ "test:integration": "vitest run --config vitest.integration.config.ts",
107
108
  "lint": "eslint src --ext .ts",
108
109
  "clean": "rm -rf dist",
109
110
  "example": "tsx examples/test-ab-experiment.ts",
@@ -1,11 +1,9 @@
1
1
  """Bundle inspection: kind detection, manifest parsing, handler loading.
2
2
 
3
- Two structurally-detected environment kinds (spec §3.1 + addendum):
4
-
5
- - **mock-tools** — ``gateway-env.toml`` at the bundle root, ``tools/*.py``
6
- handlers, ``db/schema.sql`` (+ ``db/seed.sql``), ``verifiers/*.sql``.
7
- - **harbor** — a tree of Harbor task dirs (``tasks/*/task.toml`` or
8
- ``*/task.toml``), mirroring ``benchmark_hub.harbor_adapter.detect_env_kind``.
3
+ Kind detection is :func:`gatewaysdk.environment.world.World.from_dir`; the
4
+ kinds ``gwenv`` runs in-process are **mock-tools** (``gateway-env.toml`` at the
5
+ bundle root, ``tools/*.py`` handlers, ``db/schema.sql``, ``verifiers/*.sql``)
6
+ and **harbor** (a tree of task dirs, ``tasks/*/task.toml`` or ``*/task.toml``).
9
7
 
10
8
  Handler loading executes each ``tools/*.py`` with this SDK's
11
9
  :mod:`gatewaysdk.environment.runtime` injected into ``sys.modules`` as
@@ -19,38 +17,30 @@ import functools
19
17
  import importlib.util
20
18
  import inspect
21
19
  import sys
22
- import tomllib
23
20
  import uuid
24
21
  from dataclasses import dataclass, field
25
22
  from pathlib import Path
26
23
  from typing import Any, Callable, Dict, List, Optional
27
24
 
25
+ from ._toml import tomllib
28
26
  from . import runtime
29
27
  from .errors import EnvironmentNotReady, WorldStartError
30
-
31
- KIND_MOCK_TOOLS = "mock-tools"
32
- KIND_API = "api"
33
- KIND_HARBOR = "harbor"
28
+ from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
34
29
 
35
30
 
36
31
  def detect_kind(bundle_path: Path) -> str:
37
- """"api" iff ``gateway-api-world.toml`` sits at the root; else "mock-tools"
38
- iff ``gateway-env.toml`` does; else "harbor" iff the tree holds Harbor task
39
- dirs (same glob heuristics as the runner's ``detect_env_kind``); else the
40
- bundle is not runnable here."""
32
+ """The tree's kind (``World.from_dir``), for a tree ``gwenv`` can run
33
+ in-process; a container benchmark package is not runnable here."""
41
34
  from .schema import bundle_mode
42
35
 
43
36
  bundle_mode(bundle_path) # Orphan/invalid schema declarations never downgrade.
44
- if (bundle_path / "gateway-api-world.toml").is_file():
45
- return KIND_API
46
- if (bundle_path / "gateway-env.toml").is_file():
47
- return KIND_MOCK_TOOLS
48
- if list(bundle_path.glob("tasks/*/task.toml")) or list(bundle_path.glob("*/task.toml")):
49
- return KIND_HARBOR
50
- raise EnvironmentNotReady(
51
- f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
52
- "a harbor task tree (no */task.toml) — nothing gwenv can run locally."
53
- )
37
+ world = World.from_dir(bundle_path)
38
+ if not world.local_runtime:
39
+ raise EnvironmentNotReady(
40
+ f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
41
+ "a harbor task tree (no */task.toml) — nothing gwenv can run locally."
42
+ )
43
+ return world.kind
54
44
 
55
45
 
56
46
  # ---------------------------------------------------------------------------
@@ -11,7 +11,9 @@ from tempfile import TemporaryDirectory
11
11
  from ._bundle import load_handlers, parse_manifest
12
12
  from .errors import WorldStartError
13
13
  from .schema import SQLiteStore, bundle_mode, check_bundle
14
- from .schema.validation import read_json
14
+ from .schema.batch import shape_refusal
15
+ from .schema.compiler import Contract
16
+ from .schema.validation import ContractError, fail, read_json, validate
15
17
 
16
18
 
17
19
  def materialize_template(bundle: Path, destination: Path, *, require_sql: bool = False) -> None:
@@ -19,8 +21,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
19
21
  if bundle_mode(bundle) is not None:
20
22
  contract = check_bundle(bundle)
21
23
  if destination.exists():
22
- from .schema import ContractError
23
-
24
24
  raise ContractError("state_exists", "Refusing to overwrite an existing template")
25
25
  from .schema.snapshot import current_default, default_snapshot, pin_path
26
26
 
@@ -33,8 +33,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
33
33
  if pin_path(bundle).is_file():
34
34
  # A pinned tree's data is the hosted snapshot; its data files cannot
35
35
  # stand in for it, so a missing or stale snapshot is refused, not rebuilt.
36
- from .schema import ContractError
37
-
38
36
  raise ContractError(
39
37
  "snapshot_pinned",
40
38
  "db/snapshot.json pins a hosted snapshot that is not in the tree: run the world on "
@@ -43,6 +41,7 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
43
41
  schema_path="/db/state.sqlite",
44
42
  )
45
43
  initial = read_json(bundle / "data/initial.json")
44
+ refuse_initial_rows(contract, initial)
46
45
  try:
47
46
  with SQLiteStore.create(destination, contract) as store:
48
47
  store.apply_batch(initial)
@@ -50,14 +49,10 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
50
49
  # Coverage never prunes: every change is an explicit store mutation.
51
50
  batches = bundle / "data/batches"
52
51
  if batches.is_symlink():
53
- from .schema import ContractError
54
-
55
52
  raise ContractError("path", "Refresh batches cannot be symlinks")
56
53
  if batches.exists():
57
54
  for batch in sorted(batches.iterdir()):
58
55
  if batch.is_symlink() or not batch.is_file() or batch.suffix != ".json":
59
- from .schema import ContractError
60
-
61
56
  raise ContractError("path", "Refresh journals must be regular JSON files")
62
57
  store.apply_batch(read_json(batch))
63
58
  except BaseException:
@@ -80,6 +75,55 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
80
75
  conn.close()
81
76
 
82
77
 
78
+ def refuse_initial_rows(contract: Contract, initial: object) -> None:
79
+ """Every inserted row's shape in ``data/initial.json`` against the contract,
80
+ all of them, before the store applies any: one ``ContractError`` with the
81
+ first refusal's own code, its entity, mutation index, primary key and
82
+ pointer, carrying the full ``refusals`` list — the shape the data-import
83
+ lane reports, so a hosted gate names the row and not just the rule. Rows
84
+ the store would refuse for other reasons (references, duplicate keys)
85
+ still surface from ``apply_batch`` with the mutation's index and key."""
86
+ if not isinstance(initial, dict) or not isinstance(initial.get("mutations"), list):
87
+ return
88
+ inserts = [
89
+ (index, mutation)
90
+ for index, mutation in enumerate(initial["mutations"])
91
+ if isinstance(mutation, dict) and mutation.get("op") == "insert" and isinstance(mutation.get("entity"), str)
92
+ ]
93
+ refusals = [
94
+ refused
95
+ for index, mutation in inserts
96
+ if (refused := shape_refusal(contract, 0, index, mutation["entity"], mutation.get("value"))) is not None
97
+ ]
98
+ if not refusals:
99
+ return
100
+ first = refusals[0]
101
+ summary = f"{len(refusals)} of {len(inserts)} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
102
+ context = {
103
+ "file": "data/initial.json",
104
+ "entity": first["entity"],
105
+ "index": first["index"],
106
+ "key": first["primaryKey"],
107
+ "field": first["field"],
108
+ "refused": len(refusals),
109
+ "rows": len(inserts),
110
+ "refusals": refusals,
111
+ }
112
+ row = initial["mutations"][first["index"]].get("value")
113
+ spec = contract.entities.get(first["entity"])
114
+ if spec is None:
115
+ fail("entity", summary + first["detail"], first["pointer"], **context)
116
+ if not isinstance(row, dict):
117
+ fail("rows", summary + first["detail"], first["pointer"], **context)
118
+ if any(name not in row for name in spec["primary_key"]):
119
+ fail("primary_key", summary + first["detail"], first["pointer"], **context)
120
+ try:
121
+ validate(spec["schema"], row, location=f"{spec['location']}/schema")
122
+ except ContractError as error:
123
+ raise error.contextualize(message=summary + error.detail["message"], **context)
124
+ fail("rows", summary + first["detail"], first["pointer"], **context)
125
+
126
+
83
127
  def validate_schema_materialization(bundle: Path) -> None:
84
128
  """Validate compiled artifacts and initial state without retaining a database."""
85
129
  bundle = Path(bundle)
@@ -0,0 +1,13 @@
1
+ """The one TOML reader for every Python the SDK supports.
2
+
3
+ 3.11+ ships ``tomllib``; 3.10 gets ``tomli`` (declared in pyproject as
4
+ ``tomli; python_version < '3.11'``). Import ``tomllib`` from here, never the
5
+ stdlib module directly, so the fallback lives in one place.
6
+ """
7
+
8
+ try:
9
+ import tomllib
10
+ except ModuleNotFoundError: # pragma: no cover - python 3.10
11
+ import tomli as tomllib # type: ignore[no-redef]
12
+
13
+ __all__ = ["tomllib"]
@@ -95,7 +95,6 @@ import sys
95
95
  import tempfile
96
96
  import threading
97
97
  import time
98
- import tomllib
99
98
  import urllib.parse
100
99
  import uuid
101
100
  from dataclasses import dataclass, field
@@ -104,9 +103,12 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
104
103
  from pathlib import Path
105
104
  from typing import Any, Callable
106
105
 
106
+ from .._toml import tomllib
107
107
  from . import snapshot as snapshots
108
+ from .bundle_runtime import stage_runtime_package
108
109
  from .compiler import check_bundle, compile_bundle
109
110
  from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
111
+ from .grading import keyed_entities
110
112
  from .store import SQLiteStore
111
113
  from .validation import ContractError, fail, read_json
112
114
 
@@ -719,15 +721,12 @@ class WorldState:
719
721
  def key_of(self, entity: str, row: dict) -> dict:
720
722
  return {name: row.get(name) for name in self.contract.entities[entity]["primary_key"]}
721
723
 
724
+ def fields_of(self, entity: str) -> set[str]:
725
+ schema = self.contract.entities[entity].get("schema") or {}
726
+ return set((schema.get("properties") or {}).keys())
727
+
722
728
  def dump(self) -> dict:
723
- out: dict[str, dict[str, dict]] = {}
724
- for entity, spec in self.contract.entities.items():
725
- keyed = {}
726
- for row in self.rows(entity):
727
- key = ":".join(str(row.get(field)) for field in spec["primary_key"])
728
- keyed[key] = row
729
- out[entity] = keyed
730
- return out
729
+ return keyed_entities(self.contract.entities, self.rows)
731
730
 
732
731
  def describe_snapshot(self) -> dict | None:
733
732
  """What ``/__world/state`` reports beside the rows: the snapshot in use, or null."""
@@ -791,6 +790,17 @@ def _key_filter(operation: Operation, ctx: ApiContext, entity: str) -> dict[str,
791
790
  return {name: name for name in operation.placeholders if name in keys}
792
791
 
793
792
 
793
+ def _query_field_filter(ctx: ApiContext, entity: str, sent: dict[str, list[str]]) -> dict[str, str]:
794
+ """What a read route with no ``filter`` implies from its query string: each
795
+ parameter that names a top-level field of the entity narrows to rows carrying
796
+ that value (``GET /accounts?status=frozen``), the way every vendor's list
797
+ endpoint reads. Paging parameters and names the entity does not declare are
798
+ left to the pager and the handler."""
799
+ fields = ctx.fields_of(entity)
800
+ paging = {ctx.settings.paging.limit_param, ctx.settings.paging.offset_param, ctx.settings.paging.cursor_param}
801
+ return {name: name for name in sent if name in fields and name not in paging}
802
+
803
+
794
804
  def _compare(have: str, wanted: str, mode: str) -> bool:
795
805
  if mode == "contains":
796
806
  return wanted in have
@@ -976,6 +986,9 @@ class ApiContext:
976
986
  def key_of(self, entity: str, row: dict) -> dict:
977
987
  return self.state.key_of(entity, row)
978
988
 
989
+ def fields_of(self, entity: str) -> set[str]:
990
+ return self.state.fields_of(entity)
991
+
979
992
  # -- the live feed
980
993
  def emit(self, event: dict) -> dict:
981
994
  """Record what this write changed for a page that polls (:meth:`WorldState.emit`)."""
@@ -1339,12 +1352,17 @@ def operation_request(spec: Spec, operation: Operation, args: dict) -> tuple[str
1339
1352
  for key, value in path_params.items():
1340
1353
  path = path.replace("{" + str(key) + "}", urllib.parse.quote(str(value), safe=""))
1341
1354
  query = args.get("query") if isinstance(args.get("query"), dict) else {}
1342
- if query:
1343
- path = f"{path}?" + urllib.parse.urlencode(query)
1344
1355
  body = args.get("body") if isinstance(args.get("body"), dict) else None
1345
- if body is None:
1346
- leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
1356
+ leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
1357
+ if operation.method == "GET" and leftovers:
1358
+ # A GET carries no body: a call written like a tool's (``list_accounts {"status": "frozen"}``)
1359
+ # means its query string, so the filter it asks for reaches the route.
1360
+ query = {**{k: v for k, v in leftovers.items() if not isinstance(v, (dict, list))}, **query}
1361
+ body = None
1362
+ elif body is None:
1347
1363
  body = leftovers or None
1364
+ if query:
1365
+ path = f"{path}?" + urllib.parse.urlencode({k: str(v) for k, v in query.items()})
1348
1366
  raw = json.dumps(body).encode("utf-8") if body is not None and operation.method != "GET" else b""
1349
1367
  base = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
1350
1368
  return operation.method, f"{base}{path}", raw
@@ -1627,17 +1645,19 @@ class WorldApp:
1627
1645
  if not operation.entity:
1628
1646
  return 501, {"error": "not_modelled", "operation": operation.name}
1629
1647
  entity = operation.entity
1630
- # A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
1631
- # reads as "either", not as "a" with "b" silently dropped.
1632
- values: dict[str, str | list[str]] = {
1633
- **{k: v for k, v in ctx.query_all.items() if k in operation.params},
1634
- **args,
1635
- }
1636
1648
  omit_done_cursor = ctx.settings.omits_done_cursor(op)
1637
1649
  if op.action == "create":
1638
1650
  row = ctx.insert(entity, _bind(op, ctx, body))
1639
1651
  return op.status, row if op.single else wrap([row], ctx.settings, op.envelope)
1640
1652
  filters = op.filter or _key_filter(operation, ctx, entity)
1653
+ if not op.filter and op.action == "read":
1654
+ filters = {**_query_field_filter(ctx, entity, ctx.query_all), **filters}
1655
+ # A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
1656
+ # reads as "either", not as "a" with "b" silently dropped.
1657
+ values: dict[str, str | list[str]] = {
1658
+ **{k: v for k, v in ctx.query_all.items() if k in operation.params or k in filters},
1659
+ **args,
1660
+ }
1641
1661
  if op.action in ("update", "delete") and not filters:
1642
1662
  fail(
1643
1663
  "spec",
@@ -1954,12 +1974,7 @@ def build_api_bundle(world: str | Path, out: str | Path) -> Path:
1954
1974
  f"The {spec.slug} world serving the vendor's API, seeded with its captured state. Point the provider client here.\n"
1955
1975
  )
1956
1976
  server = out_dir / "server"
1957
- package = server / "gatewaysdk" / "environment" / "schema"
1958
- package.mkdir(parents=True)
1959
- (server / "gatewaysdk" / "__init__.py").write_text("")
1960
- (server / "gatewaysdk" / "environment" / "__init__.py").write_text("")
1961
- for module in sorted(Path(__file__).parent.glob("*.py")):
1962
- shutil.copy(module, package / module.name)
1977
+ stage_runtime_package(server)
1963
1978
  shutil.copytree(root, server / "world", ignore=shutil.ignore_patterns("captures", "__pycache__", ".git"))
1964
1979
  (server / "main.py").write_text(
1965
1980
  "\n".join(
@@ -355,8 +355,11 @@ def chunk_rows(
355
355
  chunk_bytes: int = CHUNK_BYTES,
356
356
  chunk_rows: int | None = None,
357
357
  entity: str | None = None,
358
+ first_ordinal: int = 0,
358
359
  ) -> Iterator[Chunk]:
359
- """Cut a row stream into gzipped JSONL chunk files under ``workdir``.
360
+ """Cut a row stream into gzipped JSONL chunk files under ``workdir``, numbering
361
+ them from ``first_ordinal`` (the ordinals below it belong to chunks the platform
362
+ staged from the project drive).
360
363
 
361
364
  Lines are tagged ``{"entity", "row"}`` unless ``entity`` names the one entity
362
365
  every row belongs to, in which case they are bare rows. A chunk closes when the
@@ -367,8 +370,10 @@ def chunk_rows(
367
370
  workdir.mkdir(parents=True, exist_ok=True)
368
371
  if chunk_bytes < 1 or (chunk_rows is not None and chunk_rows < 1):
369
372
  fail("chunk", "chunk bounds must be positive", "/chunk")
373
+ if first_ordinal < 0:
374
+ fail("chunk", "the first ordinal must not be negative", "/chunk")
370
375
  writer: _ChunkWriter | None = None
371
- ordinal = 0
376
+ ordinal = first_ordinal
372
377
  seen = 0
373
378
  for name, row in rows:
374
379
  if entity is not None and name != entity:
@@ -391,17 +396,18 @@ def chunk_rows(
391
396
 
392
397
 
393
398
  def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tuple[str, dict]]:
394
- """Stream ``(entity, row)`` from one chunk file (gzip over JSONL; a plain file
395
- is read as is), tagged lines or bare rows of the declared entity."""
399
+ """Stream ``(entity, row)`` from one chunk file: gzip (sniffed, whatever the name)
400
+ or plain, holding any rows-file shape — the tagged or bare JSONL a CLI cut, or a
401
+ ``{entity: [rows]}`` / bare-array document a project-drive file was staged as."""
396
402
  path = Path(path)
397
403
  with open(path, "rb") as probe:
398
404
  compressed = probe.read(2) == b"\x1f\x8b"
399
405
  opener = gzip.open if compressed else open
400
406
  with opener(path, "rt", encoding="utf-8") as stream: # type: ignore[call-overload]
401
- reader = _Reader(stream)
402
- first = reader.line()
403
- if first is not None:
404
- yield from _stream_jsonl(reader, first, str(path), entity)
407
+ try:
408
+ yield from iter_file_rows(stream, str(path), entity=entity)
409
+ except UnicodeError:
410
+ fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
405
411
 
406
412
 
407
413
  # ---------------------------------------------------------------- refusals
@@ -669,9 +675,7 @@ def _apply(
669
675
  return applied, None
670
676
 
671
677
 
672
- def validate_chunks(
673
- contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None
674
- ) -> dict:
678
+ def validate_chunks(contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None) -> dict:
675
679
  """``validate-rows``: shape-only, every row of every chunk — after the batch's
676
680
  redaction hook, when it has one, so a plaintext row under ``redact=refuse``
677
681
  is refused here, in the validating phase, before anything is applied."""
@@ -0,0 +1,24 @@
1
+ """The runtime an api bundle carries beside its `server/main.py`: this package, vendored."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import shutil
6
+ from pathlib import Path
7
+
8
+ SCHEMA_PACKAGE = Path(__file__).parent
9
+ ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
10
+ # Modules outside `schema/` that its modules import; copied beside them so `main.py` resolves them.
11
+ SIBLING_MODULES = ("_toml.py",)
12
+
13
+
14
+ def stage_runtime_package(server: Path) -> Path:
15
+ """Vendor `gatewaysdk.environment.schema` (and what it imports) under `server/`; returns the package dir."""
16
+ package = server / "gatewaysdk" / "environment" / "schema"
17
+ package.mkdir(parents=True)
18
+ for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
19
+ (server / package_dir / "__init__.py").write_text("")
20
+ for name in SIBLING_MODULES:
21
+ shutil.copy(ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name)
22
+ for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
23
+ shutil.copy(module, package / module.name)
24
+ return package
@@ -8,11 +8,7 @@ from dataclasses import dataclass
8
8
  from pathlib import Path
9
9
  from typing import Any
10
10
 
11
- try:
12
- import tomllib
13
- except ModuleNotFoundError:
14
- import tomli as tomllib
15
-
11
+ from .._toml import tomllib
16
12
  from .validation import (
17
13
  LANGUAGE,
18
14
  ContractError,
@@ -51,6 +47,7 @@ FIELD_RESERVED = ("sqlite_", "gateway_", "benchmark_")
51
47
  #: An entity may not take them.
52
48
  RESERVED_STATE_KEYS = frozenset({"snapshot", "calls", "report"})
53
49
 
50
+
54
51
  def identifier(value: Any, location: str) -> str:
55
52
  if (
56
53
  not isinstance(value, str)
@@ -632,6 +629,39 @@ def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
632
629
  return contract
633
630
 
634
631
 
632
+ STALE_JOURNAL = (
633
+ "Batch targets a different domain identity/version/hash; a journal that holds "
634
+ "state is not migrated to an edited schema — re-import its rows against the new "
635
+ "contract (or restore the schema it was written against)"
636
+ )
637
+
638
+
639
+ def _journals_off_target(bundle: Path, contract: Contract) -> list[tuple[Path, str, dict]]:
640
+ """Every journal whose target is not this contract's: ``(path, location, journal)``."""
641
+ from .validation import journal_paths, read_json
642
+
643
+ off = []
644
+ for path in journal_paths(bundle):
645
+ journal = read_json(path)
646
+ location = str(path.relative_to(bundle))
647
+ if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
648
+ fail("batch", "Journal must be a batch object with a target", location)
649
+ if journal["target"] != contract.target:
650
+ off.append((path, location, journal))
651
+ return off
652
+
653
+
654
+ def stale_journals(bundle: str | Path, contract: Contract) -> list[str]:
655
+ """The journals ``compile`` will refuse: rows written against another
656
+ contract. Read-only; what ``check`` names instead of \"run compile\" when
657
+ compile is the step that cannot succeed."""
658
+ return [
659
+ location
660
+ for _path, location, journal in _journals_off_target(Path(bundle), contract)
661
+ if journal.get("mutations") != []
662
+ ]
663
+
664
+
635
665
  def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]:
636
666
  """Re-stamp every journal that holds no state onto the compiled contract's
637
667
  target, and refuse one that holds rows.
@@ -642,25 +672,11 @@ def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]
642
672
  migrate in an empty journal, whatever domain it named, so it follows the
643
673
  contract. A journal with rows is never migrated silently; the message names
644
674
  the fix. Returns the bundle-relative paths rewritten."""
645
- from .validation import journal_paths, read_json
646
-
647
675
  bundle = Path(bundle)
648
676
  retargeted = []
649
- for path in journal_paths(bundle):
650
- journal = read_json(path)
651
- location = str(path.relative_to(bundle))
652
- if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
653
- fail("batch", "Journal must be a batch object with a target", location)
654
- if journal["target"] == contract.target:
655
- continue
677
+ for path, location, journal in _journals_off_target(bundle, contract):
656
678
  if journal.get("mutations") != []:
657
- fail(
658
- "batch_target",
659
- "Batch targets a different domain identity/version/hash; a journal that holds "
660
- "state is not migrated to an edited schema — re-import its rows against the new "
661
- "contract (or restore the schema it was written against)",
662
- location,
663
- )
679
+ fail("batch_target", STALE_JOURNAL, location)
664
680
  journal["target"] = contract.target
665
681
  path.write_bytes(encoded(journal))
666
682
  retargeted.append(location)
@@ -690,18 +706,34 @@ def check_bundle(bundle: str | Path) -> Contract:
690
706
  except OSError:
691
707
  matches = False
692
708
  if not matches:
693
- fail(
694
- "lock_mismatch",
695
- "Missing, edited or incompatible contract artifact; the tree changed "
696
- "since its last compile, so run `gateway worlds schema compile <dir>` "
697
- "(state migration is not automatic)",
698
- path,
699
- )
709
+ fail("lock_mismatch", _stale_artifact_remedy(bundle, contract), path)
700
710
  from .snapshot import check_all
701
711
 
702
712
  check_all(bundle, contract)
703
713
  return contract
704
714
 
705
715
 
716
+ def _stale_artifact_remedy(bundle: Path, contract: Contract) -> str:
717
+ """What to do about a stale artifact. \"Run compile\" only when compile can
718
+ succeed; when a journal holds rows written against the previous contract,
719
+ compile refuses too, so name that step and its fix here (a check that sent
720
+ the author to a compile that failed the same way cost a dogfood session)."""
721
+ stale = stale_journals(bundle, contract)
722
+ if not stale:
723
+ return (
724
+ "Missing, edited or incompatible contract artifact; the tree changed "
725
+ "since its last compile, so run `gateway worlds schema compile <dir>` "
726
+ "(state migration is not automatic)"
727
+ )
728
+ return (
729
+ "Missing, edited or incompatible contract artifact; the schema changed since "
730
+ f"its last compile and {', '.join(stale)} holds rows written against the previous "
731
+ "contract, so `gateway worlds schema compile <dir>` will refuse too (batch_target). "
732
+ "Either restore the schema those rows were written against, or move the journal "
733
+ "aside, compile, and re-import the rows against the new contract "
734
+ "(`gateway worlds data import <dir> <rows.json>`)"
735
+ )
736
+
737
+
706
738
  def describe_bundle(bundle: str | Path) -> str:
707
739
  return build(Path(bundle)).artifacts["schema/README.md"].decode()
@@ -29,7 +29,6 @@ from __future__ import annotations
29
29
  import base64
30
30
  import json
31
31
  import threading
32
- import tomllib
33
32
  import urllib.error
34
33
  import urllib.parse
35
34
  import urllib.request
@@ -37,6 +36,7 @@ from dataclasses import dataclass, field
37
36
  from pathlib import Path
38
37
  from typing import Any
39
38
 
39
+ from .._toml import tomllib
40
40
  from .api import WorldState, load_api_settings, serve
41
41
  from .connector import SPEC, Operation, load_spec
42
42
  from .validation import fail
@@ -66,7 +66,6 @@ import re
66
66
  import shutil
67
67
  import tempfile
68
68
  import time
69
- import tomllib
70
69
  import urllib.error
71
70
  import urllib.parse
72
71
  import urllib.request
@@ -75,6 +74,7 @@ from datetime import datetime, timezone
75
74
  from pathlib import Path
76
75
  from typing import Any, Callable
77
76
 
77
+ from .._toml import tomllib
78
78
  from .compiler import check_bundle, compile_bundle
79
79
  from .store import SQLiteStore
80
80
  from .transforms import apply_transforms
@@ -0,0 +1,25 @@
1
+ # Merge `google_drive` tool results onto this world's entities. Merge answers each
2
+ # tool with one JSON document; `list_files` carries the files under `files`, in
3
+ # snake_case (the Drive contract is camelCase, hence the renames).
4
+
5
+ [ingest]
6
+ on_conflict = "latest"
7
+ unmatched = "refuse"
8
+
9
+ [[sources]]
10
+ name = "list_files"
11
+ select = "result.files[*]"
12
+ entity = "drive_files"
13
+ at = "modified_time"
14
+
15
+ [sources.fields]
16
+ id = "id"
17
+ name = "name"
18
+ mimeType = "mime_type"
19
+ starred = "starred"
20
+ trashed = "trashed"
21
+ shared = "shared"
22
+ size = "size"
23
+ webViewLink = "web_view_link"
24
+ createdTime = "created_time"
25
+ modifiedTime = "modified_time"
@@ -0,0 +1,17 @@
1
+ # How the platform fills a world on this contract from a live Merge connection
2
+ # (`gateway worlds create --template google-drive --mode live --connection <id>`).
3
+ # Each [[pulls]] is one read tool of Merge's `google_drive` tool pack, called on
4
+ # the connection's registered user; readers/merge.ingest.toml maps what comes back
5
+ # onto the world's entities. Only read tools belong here.
6
+
7
+ [reader]
8
+ transport = "merge"
9
+ provider = "google_drive"
10
+ mapping = "merge.ingest.toml"
11
+
12
+ # Files the connected account can see; trashed ones stay out of the world.
13
+ [[pulls]]
14
+ tool = "list_files"
15
+ arguments = { page_size = 100, q = "trashed = false" }
16
+ page = { cursor = "page_token", next = "next_page_token" }
17
+ max_pages = 20