@withgateway/sdk 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +4304 -2381
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4266 -2343
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +56 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +52 -27
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +280 -8
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +272 -8
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
- package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
- package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
- package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1100 -438
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1079 -424
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +45 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +41 -16
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +98 -61
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +98 -61
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +179 -25
- package/dist/worlds/index.d.ts +179 -25
- package/dist/worlds/index.js +639 -284
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +631 -276
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +2 -1
- package/runtime/gatewaysdk/environment/_bundle.py +15 -25
- package/runtime/gatewaysdk/environment/_materialize.py +53 -9
- package/runtime/gatewaysdk/environment/_toml.py +13 -0
- package/runtime/gatewaysdk/environment/schema/api.py +40 -25
- package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
- package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +156 -546
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/platform.py +30 -26
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +67 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
- package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
- package/runtime/gatewaysdk/environment/schema/tools_world.py +71 -23
- package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +507 -0
- package/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/skills/connector-schema-authoring/connector-worlds.md +8 -1
- package/skills/connector-schema-authoring/world-schema.md +1 -1
- package/skills/hooks-getting-started/SKILL.md +67 -6
- package/skills/world-data-ingestion/SKILL.md +4 -1
- package/skills/worlds-getting-started/SKILL.md +11 -11
- package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@withgateway/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.6",
|
|
4
4
|
"description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -104,6 +104,7 @@
|
|
|
104
104
|
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
105
105
|
"test": "vitest",
|
|
106
106
|
"test:run": "vitest run",
|
|
107
|
+
"test:integration": "vitest run --config vitest.integration.config.ts",
|
|
107
108
|
"lint": "eslint src --ext .ts",
|
|
108
109
|
"clean": "rm -rf dist",
|
|
109
110
|
"example": "tsx examples/test-ab-experiment.ts",
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
"""Bundle inspection: kind detection, manifest parsing, handler loading.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- **harbor** — a tree of Harbor task dirs (``tasks/*/task.toml`` or
|
|
8
|
-
``*/task.toml``), mirroring ``benchmark_hub.harbor_adapter.detect_env_kind``.
|
|
3
|
+
Kind detection is :func:`gatewaysdk.environment.world.World.from_dir`; the
|
|
4
|
+
kinds ``gwenv`` runs in-process are **mock-tools** (``gateway-env.toml`` at the
|
|
5
|
+
bundle root, ``tools/*.py`` handlers, ``db/schema.sql``, ``verifiers/*.sql``)
|
|
6
|
+
and **harbor** (a tree of task dirs, ``tasks/*/task.toml`` or ``*/task.toml``).
|
|
9
7
|
|
|
10
8
|
Handler loading executes each ``tools/*.py`` with this SDK's
|
|
11
9
|
:mod:`gatewaysdk.environment.runtime` injected into ``sys.modules`` as
|
|
@@ -19,38 +17,30 @@ import functools
|
|
|
19
17
|
import importlib.util
|
|
20
18
|
import inspect
|
|
21
19
|
import sys
|
|
22
|
-
import tomllib
|
|
23
20
|
import uuid
|
|
24
21
|
from dataclasses import dataclass, field
|
|
25
22
|
from pathlib import Path
|
|
26
23
|
from typing import Any, Callable, Dict, List, Optional
|
|
27
24
|
|
|
25
|
+
from ._toml import tomllib
|
|
28
26
|
from . import runtime
|
|
29
27
|
from .errors import EnvironmentNotReady, WorldStartError
|
|
30
|
-
|
|
31
|
-
KIND_MOCK_TOOLS = "mock-tools"
|
|
32
|
-
KIND_API = "api"
|
|
33
|
-
KIND_HARBOR = "harbor"
|
|
28
|
+
from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
|
|
34
29
|
|
|
35
30
|
|
|
36
31
|
def detect_kind(bundle_path: Path) -> str:
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
dirs (same glob heuristics as the runner's ``detect_env_kind``); else the
|
|
40
|
-
bundle is not runnable here."""
|
|
32
|
+
"""The tree's kind (``World.from_dir``), for a tree ``gwenv`` can run
|
|
33
|
+
in-process; a container benchmark package is not runnable here."""
|
|
41
34
|
from .schema import bundle_mode
|
|
42
35
|
|
|
43
36
|
bundle_mode(bundle_path) # Orphan/invalid schema declarations never downgrade.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
|
|
52
|
-
"a harbor task tree (no */task.toml) — nothing gwenv can run locally."
|
|
53
|
-
)
|
|
37
|
+
world = World.from_dir(bundle_path)
|
|
38
|
+
if not world.local_runtime:
|
|
39
|
+
raise EnvironmentNotReady(
|
|
40
|
+
f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
|
|
41
|
+
"a harbor task tree (no */task.toml) — nothing gwenv can run locally."
|
|
42
|
+
)
|
|
43
|
+
return world.kind
|
|
54
44
|
|
|
55
45
|
|
|
56
46
|
# ---------------------------------------------------------------------------
|
|
@@ -11,7 +11,9 @@ from tempfile import TemporaryDirectory
|
|
|
11
11
|
from ._bundle import load_handlers, parse_manifest
|
|
12
12
|
from .errors import WorldStartError
|
|
13
13
|
from .schema import SQLiteStore, bundle_mode, check_bundle
|
|
14
|
-
from .schema.
|
|
14
|
+
from .schema.batch import shape_refusal
|
|
15
|
+
from .schema.compiler import Contract
|
|
16
|
+
from .schema.validation import ContractError, fail, read_json, validate
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
def materialize_template(bundle: Path, destination: Path, *, require_sql: bool = False) -> None:
|
|
@@ -19,8 +21,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
19
21
|
if bundle_mode(bundle) is not None:
|
|
20
22
|
contract = check_bundle(bundle)
|
|
21
23
|
if destination.exists():
|
|
22
|
-
from .schema import ContractError
|
|
23
|
-
|
|
24
24
|
raise ContractError("state_exists", "Refusing to overwrite an existing template")
|
|
25
25
|
from .schema.snapshot import current_default, default_snapshot, pin_path
|
|
26
26
|
|
|
@@ -33,8 +33,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
33
33
|
if pin_path(bundle).is_file():
|
|
34
34
|
# A pinned tree's data is the hosted snapshot; its data files cannot
|
|
35
35
|
# stand in for it, so a missing or stale snapshot is refused, not rebuilt.
|
|
36
|
-
from .schema import ContractError
|
|
37
|
-
|
|
38
36
|
raise ContractError(
|
|
39
37
|
"snapshot_pinned",
|
|
40
38
|
"db/snapshot.json pins a hosted snapshot that is not in the tree: run the world on "
|
|
@@ -43,6 +41,7 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
43
41
|
schema_path="/db/state.sqlite",
|
|
44
42
|
)
|
|
45
43
|
initial = read_json(bundle / "data/initial.json")
|
|
44
|
+
refuse_initial_rows(contract, initial)
|
|
46
45
|
try:
|
|
47
46
|
with SQLiteStore.create(destination, contract) as store:
|
|
48
47
|
store.apply_batch(initial)
|
|
@@ -50,14 +49,10 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
50
49
|
# Coverage never prunes: every change is an explicit store mutation.
|
|
51
50
|
batches = bundle / "data/batches"
|
|
52
51
|
if batches.is_symlink():
|
|
53
|
-
from .schema import ContractError
|
|
54
|
-
|
|
55
52
|
raise ContractError("path", "Refresh batches cannot be symlinks")
|
|
56
53
|
if batches.exists():
|
|
57
54
|
for batch in sorted(batches.iterdir()):
|
|
58
55
|
if batch.is_symlink() or not batch.is_file() or batch.suffix != ".json":
|
|
59
|
-
from .schema import ContractError
|
|
60
|
-
|
|
61
56
|
raise ContractError("path", "Refresh journals must be regular JSON files")
|
|
62
57
|
store.apply_batch(read_json(batch))
|
|
63
58
|
except BaseException:
|
|
@@ -80,6 +75,55 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
80
75
|
conn.close()
|
|
81
76
|
|
|
82
77
|
|
|
78
|
+
def refuse_initial_rows(contract: Contract, initial: object) -> None:
|
|
79
|
+
"""Every inserted row's shape in ``data/initial.json`` against the contract,
|
|
80
|
+
all of them, before the store applies any: one ``ContractError`` with the
|
|
81
|
+
first refusal's own code, its entity, mutation index, primary key and
|
|
82
|
+
pointer, carrying the full ``refusals`` list — the shape the data-import
|
|
83
|
+
lane reports, so a hosted gate names the row and not just the rule. Rows
|
|
84
|
+
the store would refuse for other reasons (references, duplicate keys)
|
|
85
|
+
still surface from ``apply_batch`` with the mutation's index and key."""
|
|
86
|
+
if not isinstance(initial, dict) or not isinstance(initial.get("mutations"), list):
|
|
87
|
+
return
|
|
88
|
+
inserts = [
|
|
89
|
+
(index, mutation)
|
|
90
|
+
for index, mutation in enumerate(initial["mutations"])
|
|
91
|
+
if isinstance(mutation, dict) and mutation.get("op") == "insert" and isinstance(mutation.get("entity"), str)
|
|
92
|
+
]
|
|
93
|
+
refusals = [
|
|
94
|
+
refused
|
|
95
|
+
for index, mutation in inserts
|
|
96
|
+
if (refused := shape_refusal(contract, 0, index, mutation["entity"], mutation.get("value"))) is not None
|
|
97
|
+
]
|
|
98
|
+
if not refusals:
|
|
99
|
+
return
|
|
100
|
+
first = refusals[0]
|
|
101
|
+
summary = f"{len(refusals)} of {len(inserts)} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
|
|
102
|
+
context = {
|
|
103
|
+
"file": "data/initial.json",
|
|
104
|
+
"entity": first["entity"],
|
|
105
|
+
"index": first["index"],
|
|
106
|
+
"key": first["primaryKey"],
|
|
107
|
+
"field": first["field"],
|
|
108
|
+
"refused": len(refusals),
|
|
109
|
+
"rows": len(inserts),
|
|
110
|
+
"refusals": refusals,
|
|
111
|
+
}
|
|
112
|
+
row = initial["mutations"][first["index"]].get("value")
|
|
113
|
+
spec = contract.entities.get(first["entity"])
|
|
114
|
+
if spec is None:
|
|
115
|
+
fail("entity", summary + first["detail"], first["pointer"], **context)
|
|
116
|
+
if not isinstance(row, dict):
|
|
117
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
118
|
+
if any(name not in row for name in spec["primary_key"]):
|
|
119
|
+
fail("primary_key", summary + first["detail"], first["pointer"], **context)
|
|
120
|
+
try:
|
|
121
|
+
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
122
|
+
except ContractError as error:
|
|
123
|
+
raise error.contextualize(message=summary + error.detail["message"], **context)
|
|
124
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
125
|
+
|
|
126
|
+
|
|
83
127
|
def validate_schema_materialization(bundle: Path) -> None:
|
|
84
128
|
"""Validate compiled artifacts and initial state without retaining a database."""
|
|
85
129
|
bundle = Path(bundle)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""The one TOML reader for every Python the SDK supports.
|
|
2
|
+
|
|
3
|
+
3.11+ ships ``tomllib``; 3.10 gets ``tomli`` (declared in pyproject as
|
|
4
|
+
``tomli; python_version < '3.11'``). Import ``tomllib`` from here, never the
|
|
5
|
+
stdlib module directly, so the fallback lives in one place.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
import tomllib
|
|
10
|
+
except ModuleNotFoundError: # pragma: no cover - python 3.10
|
|
11
|
+
import tomli as tomllib # type: ignore[no-redef]
|
|
12
|
+
|
|
13
|
+
__all__ = ["tomllib"]
|
|
@@ -95,7 +95,6 @@ import sys
|
|
|
95
95
|
import tempfile
|
|
96
96
|
import threading
|
|
97
97
|
import time
|
|
98
|
-
import tomllib
|
|
99
98
|
import urllib.parse
|
|
100
99
|
import uuid
|
|
101
100
|
from dataclasses import dataclass, field
|
|
@@ -104,9 +103,12 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
104
103
|
from pathlib import Path
|
|
105
104
|
from typing import Any, Callable
|
|
106
105
|
|
|
106
|
+
from .._toml import tomllib
|
|
107
107
|
from . import snapshot as snapshots
|
|
108
|
+
from .bundle_runtime import stage_runtime_package
|
|
108
109
|
from .compiler import check_bundle, compile_bundle
|
|
109
110
|
from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
|
|
111
|
+
from .grading import keyed_entities
|
|
110
112
|
from .store import SQLiteStore
|
|
111
113
|
from .validation import ContractError, fail, read_json
|
|
112
114
|
|
|
@@ -719,15 +721,12 @@ class WorldState:
|
|
|
719
721
|
def key_of(self, entity: str, row: dict) -> dict:
|
|
720
722
|
return {name: row.get(name) for name in self.contract.entities[entity]["primary_key"]}
|
|
721
723
|
|
|
724
|
+
def fields_of(self, entity: str) -> set[str]:
|
|
725
|
+
schema = self.contract.entities[entity].get("schema") or {}
|
|
726
|
+
return set((schema.get("properties") or {}).keys())
|
|
727
|
+
|
|
722
728
|
def dump(self) -> dict:
|
|
723
|
-
|
|
724
|
-
for entity, spec in self.contract.entities.items():
|
|
725
|
-
keyed = {}
|
|
726
|
-
for row in self.rows(entity):
|
|
727
|
-
key = ":".join(str(row.get(field)) for field in spec["primary_key"])
|
|
728
|
-
keyed[key] = row
|
|
729
|
-
out[entity] = keyed
|
|
730
|
-
return out
|
|
729
|
+
return keyed_entities(self.contract.entities, self.rows)
|
|
731
730
|
|
|
732
731
|
def describe_snapshot(self) -> dict | None:
|
|
733
732
|
"""What ``/__world/state`` reports beside the rows: the snapshot in use, or null."""
|
|
@@ -791,6 +790,17 @@ def _key_filter(operation: Operation, ctx: ApiContext, entity: str) -> dict[str,
|
|
|
791
790
|
return {name: name for name in operation.placeholders if name in keys}
|
|
792
791
|
|
|
793
792
|
|
|
793
|
+
def _query_field_filter(ctx: ApiContext, entity: str, sent: dict[str, list[str]]) -> dict[str, str]:
|
|
794
|
+
"""What a read route with no ``filter`` implies from its query string: each
|
|
795
|
+
parameter that names a top-level field of the entity narrows to rows carrying
|
|
796
|
+
that value (``GET /accounts?status=frozen``), the way every vendor's list
|
|
797
|
+
endpoint reads. Paging parameters and names the entity does not declare are
|
|
798
|
+
left to the pager and the handler."""
|
|
799
|
+
fields = ctx.fields_of(entity)
|
|
800
|
+
paging = {ctx.settings.paging.limit_param, ctx.settings.paging.offset_param, ctx.settings.paging.cursor_param}
|
|
801
|
+
return {name: name for name in sent if name in fields and name not in paging}
|
|
802
|
+
|
|
803
|
+
|
|
794
804
|
def _compare(have: str, wanted: str, mode: str) -> bool:
|
|
795
805
|
if mode == "contains":
|
|
796
806
|
return wanted in have
|
|
@@ -976,6 +986,9 @@ class ApiContext:
|
|
|
976
986
|
def key_of(self, entity: str, row: dict) -> dict:
|
|
977
987
|
return self.state.key_of(entity, row)
|
|
978
988
|
|
|
989
|
+
def fields_of(self, entity: str) -> set[str]:
|
|
990
|
+
return self.state.fields_of(entity)
|
|
991
|
+
|
|
979
992
|
# -- the live feed
|
|
980
993
|
def emit(self, event: dict) -> dict:
|
|
981
994
|
"""Record what this write changed for a page that polls (:meth:`WorldState.emit`)."""
|
|
@@ -1339,12 +1352,17 @@ def operation_request(spec: Spec, operation: Operation, args: dict) -> tuple[str
|
|
|
1339
1352
|
for key, value in path_params.items():
|
|
1340
1353
|
path = path.replace("{" + str(key) + "}", urllib.parse.quote(str(value), safe=""))
|
|
1341
1354
|
query = args.get("query") if isinstance(args.get("query"), dict) else {}
|
|
1342
|
-
if query:
|
|
1343
|
-
path = f"{path}?" + urllib.parse.urlencode(query)
|
|
1344
1355
|
body = args.get("body") if isinstance(args.get("body"), dict) else None
|
|
1345
|
-
if body
|
|
1346
|
-
|
|
1356
|
+
leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
|
|
1357
|
+
if operation.method == "GET" and leftovers:
|
|
1358
|
+
# A GET carries no body: a call written like a tool's (``list_accounts {"status": "frozen"}``)
|
|
1359
|
+
# means its query string, so the filter it asks for reaches the route.
|
|
1360
|
+
query = {**{k: v for k, v in leftovers.items() if not isinstance(v, (dict, list))}, **query}
|
|
1361
|
+
body = None
|
|
1362
|
+
elif body is None:
|
|
1347
1363
|
body = leftovers or None
|
|
1364
|
+
if query:
|
|
1365
|
+
path = f"{path}?" + urllib.parse.urlencode({k: str(v) for k, v in query.items()})
|
|
1348
1366
|
raw = json.dumps(body).encode("utf-8") if body is not None and operation.method != "GET" else b""
|
|
1349
1367
|
base = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
|
|
1350
1368
|
return operation.method, f"{base}{path}", raw
|
|
@@ -1627,17 +1645,19 @@ class WorldApp:
|
|
|
1627
1645
|
if not operation.entity:
|
|
1628
1646
|
return 501, {"error": "not_modelled", "operation": operation.name}
|
|
1629
1647
|
entity = operation.entity
|
|
1630
|
-
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
1631
|
-
# reads as "either", not as "a" with "b" silently dropped.
|
|
1632
|
-
values: dict[str, str | list[str]] = {
|
|
1633
|
-
**{k: v for k, v in ctx.query_all.items() if k in operation.params},
|
|
1634
|
-
**args,
|
|
1635
|
-
}
|
|
1636
1648
|
omit_done_cursor = ctx.settings.omits_done_cursor(op)
|
|
1637
1649
|
if op.action == "create":
|
|
1638
1650
|
row = ctx.insert(entity, _bind(op, ctx, body))
|
|
1639
1651
|
return op.status, row if op.single else wrap([row], ctx.settings, op.envelope)
|
|
1640
1652
|
filters = op.filter or _key_filter(operation, ctx, entity)
|
|
1653
|
+
if not op.filter and op.action == "read":
|
|
1654
|
+
filters = {**_query_field_filter(ctx, entity, ctx.query_all), **filters}
|
|
1655
|
+
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
1656
|
+
# reads as "either", not as "a" with "b" silently dropped.
|
|
1657
|
+
values: dict[str, str | list[str]] = {
|
|
1658
|
+
**{k: v for k, v in ctx.query_all.items() if k in operation.params or k in filters},
|
|
1659
|
+
**args,
|
|
1660
|
+
}
|
|
1641
1661
|
if op.action in ("update", "delete") and not filters:
|
|
1642
1662
|
fail(
|
|
1643
1663
|
"spec",
|
|
@@ -1954,12 +1974,7 @@ def build_api_bundle(world: str | Path, out: str | Path) -> Path:
|
|
|
1954
1974
|
f"The {spec.slug} world serving the vendor's API, seeded with its captured state. Point the provider client here.\n"
|
|
1955
1975
|
)
|
|
1956
1976
|
server = out_dir / "server"
|
|
1957
|
-
|
|
1958
|
-
package.mkdir(parents=True)
|
|
1959
|
-
(server / "gatewaysdk" / "__init__.py").write_text("")
|
|
1960
|
-
(server / "gatewaysdk" / "environment" / "__init__.py").write_text("")
|
|
1961
|
-
for module in sorted(Path(__file__).parent.glob("*.py")):
|
|
1962
|
-
shutil.copy(module, package / module.name)
|
|
1977
|
+
stage_runtime_package(server)
|
|
1963
1978
|
shutil.copytree(root, server / "world", ignore=shutil.ignore_patterns("captures", "__pycache__", ".git"))
|
|
1964
1979
|
(server / "main.py").write_text(
|
|
1965
1980
|
"\n".join(
|
|
@@ -355,8 +355,11 @@ def chunk_rows(
|
|
|
355
355
|
chunk_bytes: int = CHUNK_BYTES,
|
|
356
356
|
chunk_rows: int | None = None,
|
|
357
357
|
entity: str | None = None,
|
|
358
|
+
first_ordinal: int = 0,
|
|
358
359
|
) -> Iterator[Chunk]:
|
|
359
|
-
"""Cut a row stream into gzipped JSONL chunk files under ``workdir
|
|
360
|
+
"""Cut a row stream into gzipped JSONL chunk files under ``workdir``, numbering
|
|
361
|
+
them from ``first_ordinal`` (the ordinals below it belong to chunks the platform
|
|
362
|
+
staged from the project drive).
|
|
360
363
|
|
|
361
364
|
Lines are tagged ``{"entity", "row"}`` unless ``entity`` names the one entity
|
|
362
365
|
every row belongs to, in which case they are bare rows. A chunk closes when the
|
|
@@ -367,8 +370,10 @@ def chunk_rows(
|
|
|
367
370
|
workdir.mkdir(parents=True, exist_ok=True)
|
|
368
371
|
if chunk_bytes < 1 or (chunk_rows is not None and chunk_rows < 1):
|
|
369
372
|
fail("chunk", "chunk bounds must be positive", "/chunk")
|
|
373
|
+
if first_ordinal < 0:
|
|
374
|
+
fail("chunk", "the first ordinal must not be negative", "/chunk")
|
|
370
375
|
writer: _ChunkWriter | None = None
|
|
371
|
-
ordinal =
|
|
376
|
+
ordinal = first_ordinal
|
|
372
377
|
seen = 0
|
|
373
378
|
for name, row in rows:
|
|
374
379
|
if entity is not None and name != entity:
|
|
@@ -391,17 +396,18 @@ def chunk_rows(
|
|
|
391
396
|
|
|
392
397
|
|
|
393
398
|
def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tuple[str, dict]]:
|
|
394
|
-
"""Stream ``(entity, row)`` from one chunk file
|
|
395
|
-
|
|
399
|
+
"""Stream ``(entity, row)`` from one chunk file: gzip (sniffed, whatever the name)
|
|
400
|
+
or plain, holding any rows-file shape — the tagged or bare JSONL a CLI cut, or a
|
|
401
|
+
``{entity: [rows]}`` / bare-array document a project-drive file was staged as."""
|
|
396
402
|
path = Path(path)
|
|
397
403
|
with open(path, "rb") as probe:
|
|
398
404
|
compressed = probe.read(2) == b"\x1f\x8b"
|
|
399
405
|
opener = gzip.open if compressed else open
|
|
400
406
|
with opener(path, "rt", encoding="utf-8") as stream: # type: ignore[call-overload]
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
407
|
+
try:
|
|
408
|
+
yield from iter_file_rows(stream, str(path), entity=entity)
|
|
409
|
+
except UnicodeError:
|
|
410
|
+
fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
|
|
405
411
|
|
|
406
412
|
|
|
407
413
|
# ---------------------------------------------------------------- refusals
|
|
@@ -669,9 +675,7 @@ def _apply(
|
|
|
669
675
|
return applied, None
|
|
670
676
|
|
|
671
677
|
|
|
672
|
-
def validate_chunks(
|
|
673
|
-
contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None
|
|
674
|
-
) -> dict:
|
|
678
|
+
def validate_chunks(contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None) -> dict:
|
|
675
679
|
"""``validate-rows``: shape-only, every row of every chunk — after the batch's
|
|
676
680
|
redaction hook, when it has one, so a plaintext row under ``redact=refuse``
|
|
677
681
|
is refused here, in the validating phase, before anything is applied."""
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The runtime an api bundle carries beside its `server/main.py`: this package, vendored."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
SCHEMA_PACKAGE = Path(__file__).parent
|
|
9
|
+
ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
|
|
10
|
+
# Modules outside `schema/` that its modules import; copied beside them so `main.py` resolves them.
|
|
11
|
+
SIBLING_MODULES = ("_toml.py",)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def stage_runtime_package(server: Path) -> Path:
|
|
15
|
+
"""Vendor `gatewaysdk.environment.schema` (and what it imports) under `server/`; returns the package dir."""
|
|
16
|
+
package = server / "gatewaysdk" / "environment" / "schema"
|
|
17
|
+
package.mkdir(parents=True)
|
|
18
|
+
for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
|
|
19
|
+
(server / package_dir / "__init__.py").write_text("")
|
|
20
|
+
for name in SIBLING_MODULES:
|
|
21
|
+
shutil.copy(ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name)
|
|
22
|
+
for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
|
|
23
|
+
shutil.copy(module, package / module.name)
|
|
24
|
+
return package
|
|
@@ -8,11 +8,7 @@ from dataclasses import dataclass
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
import tomllib
|
|
13
|
-
except ModuleNotFoundError:
|
|
14
|
-
import tomli as tomllib
|
|
15
|
-
|
|
11
|
+
from .._toml import tomllib
|
|
16
12
|
from .validation import (
|
|
17
13
|
LANGUAGE,
|
|
18
14
|
ContractError,
|
|
@@ -51,6 +47,7 @@ FIELD_RESERVED = ("sqlite_", "gateway_", "benchmark_")
|
|
|
51
47
|
#: An entity may not take them.
|
|
52
48
|
RESERVED_STATE_KEYS = frozenset({"snapshot", "calls", "report"})
|
|
53
49
|
|
|
50
|
+
|
|
54
51
|
def identifier(value: Any, location: str) -> str:
|
|
55
52
|
if (
|
|
56
53
|
not isinstance(value, str)
|
|
@@ -632,6 +629,39 @@ def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
|
|
|
632
629
|
return contract
|
|
633
630
|
|
|
634
631
|
|
|
632
|
+
STALE_JOURNAL = (
|
|
633
|
+
"Batch targets a different domain identity/version/hash; a journal that holds "
|
|
634
|
+
"state is not migrated to an edited schema — re-import its rows against the new "
|
|
635
|
+
"contract (or restore the schema it was written against)"
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _journals_off_target(bundle: Path, contract: Contract) -> list[tuple[Path, str, dict]]:
|
|
640
|
+
"""Every journal whose target is not this contract's: ``(path, location, journal)``."""
|
|
641
|
+
from .validation import journal_paths, read_json
|
|
642
|
+
|
|
643
|
+
off = []
|
|
644
|
+
for path in journal_paths(bundle):
|
|
645
|
+
journal = read_json(path)
|
|
646
|
+
location = str(path.relative_to(bundle))
|
|
647
|
+
if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
|
|
648
|
+
fail("batch", "Journal must be a batch object with a target", location)
|
|
649
|
+
if journal["target"] != contract.target:
|
|
650
|
+
off.append((path, location, journal))
|
|
651
|
+
return off
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def stale_journals(bundle: str | Path, contract: Contract) -> list[str]:
|
|
655
|
+
"""The journals ``compile`` will refuse: rows written against another
|
|
656
|
+
contract. Read-only; what ``check`` names instead of \"run compile\" when
|
|
657
|
+
compile is the step that cannot succeed."""
|
|
658
|
+
return [
|
|
659
|
+
location
|
|
660
|
+
for _path, location, journal in _journals_off_target(Path(bundle), contract)
|
|
661
|
+
if journal.get("mutations") != []
|
|
662
|
+
]
|
|
663
|
+
|
|
664
|
+
|
|
635
665
|
def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]:
|
|
636
666
|
"""Re-stamp every journal that holds no state onto the compiled contract's
|
|
637
667
|
target, and refuse one that holds rows.
|
|
@@ -642,25 +672,11 @@ def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]
|
|
|
642
672
|
migrate in an empty journal, whatever domain it named, so it follows the
|
|
643
673
|
contract. A journal with rows is never migrated silently; the message names
|
|
644
674
|
the fix. Returns the bundle-relative paths rewritten."""
|
|
645
|
-
from .validation import journal_paths, read_json
|
|
646
|
-
|
|
647
675
|
bundle = Path(bundle)
|
|
648
676
|
retargeted = []
|
|
649
|
-
for path in
|
|
650
|
-
journal = read_json(path)
|
|
651
|
-
location = str(path.relative_to(bundle))
|
|
652
|
-
if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
|
|
653
|
-
fail("batch", "Journal must be a batch object with a target", location)
|
|
654
|
-
if journal["target"] == contract.target:
|
|
655
|
-
continue
|
|
677
|
+
for path, location, journal in _journals_off_target(bundle, contract):
|
|
656
678
|
if journal.get("mutations") != []:
|
|
657
|
-
fail(
|
|
658
|
-
"batch_target",
|
|
659
|
-
"Batch targets a different domain identity/version/hash; a journal that holds "
|
|
660
|
-
"state is not migrated to an edited schema — re-import its rows against the new "
|
|
661
|
-
"contract (or restore the schema it was written against)",
|
|
662
|
-
location,
|
|
663
|
-
)
|
|
679
|
+
fail("batch_target", STALE_JOURNAL, location)
|
|
664
680
|
journal["target"] = contract.target
|
|
665
681
|
path.write_bytes(encoded(journal))
|
|
666
682
|
retargeted.append(location)
|
|
@@ -690,18 +706,34 @@ def check_bundle(bundle: str | Path) -> Contract:
|
|
|
690
706
|
except OSError:
|
|
691
707
|
matches = False
|
|
692
708
|
if not matches:
|
|
693
|
-
fail(
|
|
694
|
-
"lock_mismatch",
|
|
695
|
-
"Missing, edited or incompatible contract artifact; the tree changed "
|
|
696
|
-
"since its last compile, so run `gateway worlds schema compile <dir>` "
|
|
697
|
-
"(state migration is not automatic)",
|
|
698
|
-
path,
|
|
699
|
-
)
|
|
709
|
+
fail("lock_mismatch", _stale_artifact_remedy(bundle, contract), path)
|
|
700
710
|
from .snapshot import check_all
|
|
701
711
|
|
|
702
712
|
check_all(bundle, contract)
|
|
703
713
|
return contract
|
|
704
714
|
|
|
705
715
|
|
|
716
|
+
def _stale_artifact_remedy(bundle: Path, contract: Contract) -> str:
|
|
717
|
+
"""What to do about a stale artifact. \"Run compile\" only when compile can
|
|
718
|
+
succeed; when a journal holds rows written against the previous contract,
|
|
719
|
+
compile refuses too, so name that step and its fix here (a check that sent
|
|
720
|
+
the author to a compile that failed the same way cost a dogfood session)."""
|
|
721
|
+
stale = stale_journals(bundle, contract)
|
|
722
|
+
if not stale:
|
|
723
|
+
return (
|
|
724
|
+
"Missing, edited or incompatible contract artifact; the tree changed "
|
|
725
|
+
"since its last compile, so run `gateway worlds schema compile <dir>` "
|
|
726
|
+
"(state migration is not automatic)"
|
|
727
|
+
)
|
|
728
|
+
return (
|
|
729
|
+
"Missing, edited or incompatible contract artifact; the schema changed since "
|
|
730
|
+
f"its last compile and {', '.join(stale)} holds rows written against the previous "
|
|
731
|
+
"contract, so `gateway worlds schema compile <dir>` will refuse too (batch_target). "
|
|
732
|
+
"Either restore the schema those rows were written against, or move the journal "
|
|
733
|
+
"aside, compile, and re-import the rows against the new contract "
|
|
734
|
+
"(`gateway worlds data import <dir> <rows.json>`)"
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
|
|
706
738
|
def describe_bundle(bundle: str | Path) -> str:
|
|
707
739
|
return build(Path(bundle)).artifacts["schema/README.md"].decode()
|
|
@@ -29,7 +29,6 @@ from __future__ import annotations
|
|
|
29
29
|
import base64
|
|
30
30
|
import json
|
|
31
31
|
import threading
|
|
32
|
-
import tomllib
|
|
33
32
|
import urllib.error
|
|
34
33
|
import urllib.parse
|
|
35
34
|
import urllib.request
|
|
@@ -37,6 +36,7 @@ from dataclasses import dataclass, field
|
|
|
37
36
|
from pathlib import Path
|
|
38
37
|
from typing import Any
|
|
39
38
|
|
|
39
|
+
from .._toml import tomllib
|
|
40
40
|
from .api import WorldState, load_api_settings, serve
|
|
41
41
|
from .connector import SPEC, Operation, load_spec
|
|
42
42
|
from .validation import fail
|
|
@@ -66,7 +66,6 @@ import re
|
|
|
66
66
|
import shutil
|
|
67
67
|
import tempfile
|
|
68
68
|
import time
|
|
69
|
-
import tomllib
|
|
70
69
|
import urllib.error
|
|
71
70
|
import urllib.parse
|
|
72
71
|
import urllib.request
|
|
@@ -75,6 +74,7 @@ from datetime import datetime, timezone
|
|
|
75
74
|
from pathlib import Path
|
|
76
75
|
from typing import Any, Callable
|
|
77
76
|
|
|
77
|
+
from .._toml import tomllib
|
|
78
78
|
from .compiler import check_bundle, compile_bundle
|
|
79
79
|
from .store import SQLiteStore
|
|
80
80
|
from .transforms import apply_transforms
|
package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Merge `google_drive` tool results onto this world's entities. Merge answers each
|
|
2
|
+
# tool with one JSON document; `list_files` carries the files under `files`, in
|
|
3
|
+
# snake_case (the Drive contract is camelCase, hence the renames).
|
|
4
|
+
|
|
5
|
+
[ingest]
|
|
6
|
+
on_conflict = "latest"
|
|
7
|
+
unmatched = "refuse"
|
|
8
|
+
|
|
9
|
+
[[sources]]
|
|
10
|
+
name = "list_files"
|
|
11
|
+
select = "result.files[*]"
|
|
12
|
+
entity = "drive_files"
|
|
13
|
+
at = "modified_time"
|
|
14
|
+
|
|
15
|
+
[sources.fields]
|
|
16
|
+
id = "id"
|
|
17
|
+
name = "name"
|
|
18
|
+
mimeType = "mime_type"
|
|
19
|
+
starred = "starred"
|
|
20
|
+
trashed = "trashed"
|
|
21
|
+
shared = "shared"
|
|
22
|
+
size = "size"
|
|
23
|
+
webViewLink = "web_view_link"
|
|
24
|
+
createdTime = "created_time"
|
|
25
|
+
modifiedTime = "modified_time"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# How the platform fills a world on this contract from a live Merge connection
|
|
2
|
+
# (`gateway worlds create --template google-drive --mode live --connection <id>`).
|
|
3
|
+
# Each [[pulls]] is one read tool of Merge's `google_drive` tool pack, called on
|
|
4
|
+
# the connection's registered user; readers/merge.ingest.toml maps what comes back
|
|
5
|
+
# onto the world's entities. Only read tools belong here.
|
|
6
|
+
|
|
7
|
+
[reader]
|
|
8
|
+
transport = "merge"
|
|
9
|
+
provider = "google_drive"
|
|
10
|
+
mapping = "merge.ingest.toml"
|
|
11
|
+
|
|
12
|
+
# Files the connected account can see; trashed ones stay out of the world.
|
|
13
|
+
[[pulls]]
|
|
14
|
+
tool = "list_files"
|
|
15
|
+
arguments = { page_size = 100, q = "trashed = false" }
|
|
16
|
+
page = { cursor = "page_token", next = "next_page_token" }
|
|
17
|
+
max_pages = 20
|