@withgateway/sdk 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3121 -1283
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3185 -1347
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +9 -0
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +9 -0
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +280 -8
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +272 -8
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
- package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
- package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
- package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +726 -167
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +718 -167
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +9 -0
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +9 -0
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +97 -61
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +97 -61
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +154 -4
- package/dist/worlds/index.d.ts +154 -4
- package/dist/worlds/index.js +348 -97
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +348 -97
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +53 -9
- package/runtime/gatewaysdk/environment/_toml.py +13 -0
- package/runtime/gatewaysdk/environment/schema/api.py +3 -7
- package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
- package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/platform.py +16 -4
- package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
- package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +61 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
- package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
- package/runtime/gatewaysdk/environment/schema/tools_world.py +1 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
- package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +120 -4
- package/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/skills/connector-schema-authoring/connector-worlds.md +8 -1
- package/skills/connector-schema-authoring/world-schema.md +1 -1
- package/skills/hooks-getting-started/SKILL.md +61 -2
- package/skills/world-data-ingestion/SKILL.md +4 -1
- package/skills/worlds-getting-started/SKILL.md +11 -11
- package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/package.json
CHANGED
|
@@ -17,12 +17,12 @@ import functools
|
|
|
17
17
|
import importlib.util
|
|
18
18
|
import inspect
|
|
19
19
|
import sys
|
|
20
|
-
import tomllib
|
|
21
20
|
import uuid
|
|
22
21
|
from dataclasses import dataclass, field
|
|
23
22
|
from pathlib import Path
|
|
24
23
|
from typing import Any, Callable, Dict, List, Optional
|
|
25
24
|
|
|
25
|
+
from ._toml import tomllib
|
|
26
26
|
from . import runtime
|
|
27
27
|
from .errors import EnvironmentNotReady, WorldStartError
|
|
28
28
|
from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
|
|
@@ -11,7 +11,9 @@ from tempfile import TemporaryDirectory
|
|
|
11
11
|
from ._bundle import load_handlers, parse_manifest
|
|
12
12
|
from .errors import WorldStartError
|
|
13
13
|
from .schema import SQLiteStore, bundle_mode, check_bundle
|
|
14
|
-
from .schema.
|
|
14
|
+
from .schema.batch import shape_refusal
|
|
15
|
+
from .schema.compiler import Contract
|
|
16
|
+
from .schema.validation import ContractError, fail, read_json, validate
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
def materialize_template(bundle: Path, destination: Path, *, require_sql: bool = False) -> None:
|
|
@@ -19,8 +21,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
19
21
|
if bundle_mode(bundle) is not None:
|
|
20
22
|
contract = check_bundle(bundle)
|
|
21
23
|
if destination.exists():
|
|
22
|
-
from .schema import ContractError
|
|
23
|
-
|
|
24
24
|
raise ContractError("state_exists", "Refusing to overwrite an existing template")
|
|
25
25
|
from .schema.snapshot import current_default, default_snapshot, pin_path
|
|
26
26
|
|
|
@@ -33,8 +33,6 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
33
33
|
if pin_path(bundle).is_file():
|
|
34
34
|
# A pinned tree's data is the hosted snapshot; its data files cannot
|
|
35
35
|
# stand in for it, so a missing or stale snapshot is refused, not rebuilt.
|
|
36
|
-
from .schema import ContractError
|
|
37
|
-
|
|
38
36
|
raise ContractError(
|
|
39
37
|
"snapshot_pinned",
|
|
40
38
|
"db/snapshot.json pins a hosted snapshot that is not in the tree: run the world on "
|
|
@@ -43,6 +41,7 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
43
41
|
schema_path="/db/state.sqlite",
|
|
44
42
|
)
|
|
45
43
|
initial = read_json(bundle / "data/initial.json")
|
|
44
|
+
refuse_initial_rows(contract, initial)
|
|
46
45
|
try:
|
|
47
46
|
with SQLiteStore.create(destination, contract) as store:
|
|
48
47
|
store.apply_batch(initial)
|
|
@@ -50,14 +49,10 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
50
49
|
# Coverage never prunes: every change is an explicit store mutation.
|
|
51
50
|
batches = bundle / "data/batches"
|
|
52
51
|
if batches.is_symlink():
|
|
53
|
-
from .schema import ContractError
|
|
54
|
-
|
|
55
52
|
raise ContractError("path", "Refresh batches cannot be symlinks")
|
|
56
53
|
if batches.exists():
|
|
57
54
|
for batch in sorted(batches.iterdir()):
|
|
58
55
|
if batch.is_symlink() or not batch.is_file() or batch.suffix != ".json":
|
|
59
|
-
from .schema import ContractError
|
|
60
|
-
|
|
61
56
|
raise ContractError("path", "Refresh journals must be regular JSON files")
|
|
62
57
|
store.apply_batch(read_json(batch))
|
|
63
58
|
except BaseException:
|
|
@@ -80,6 +75,55 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
80
75
|
conn.close()
|
|
81
76
|
|
|
82
77
|
|
|
78
|
+
def refuse_initial_rows(contract: Contract, initial: object) -> None:
|
|
79
|
+
"""Every inserted row's shape in ``data/initial.json`` against the contract,
|
|
80
|
+
all of them, before the store applies any: one ``ContractError`` with the
|
|
81
|
+
first refusal's own code, its entity, mutation index, primary key and
|
|
82
|
+
pointer, carrying the full ``refusals`` list — the shape the data-import
|
|
83
|
+
lane reports, so a hosted gate names the row and not just the rule. Rows
|
|
84
|
+
the store would refuse for other reasons (references, duplicate keys)
|
|
85
|
+
still surface from ``apply_batch`` with the mutation's index and key."""
|
|
86
|
+
if not isinstance(initial, dict) or not isinstance(initial.get("mutations"), list):
|
|
87
|
+
return
|
|
88
|
+
inserts = [
|
|
89
|
+
(index, mutation)
|
|
90
|
+
for index, mutation in enumerate(initial["mutations"])
|
|
91
|
+
if isinstance(mutation, dict) and mutation.get("op") == "insert" and isinstance(mutation.get("entity"), str)
|
|
92
|
+
]
|
|
93
|
+
refusals = [
|
|
94
|
+
refused
|
|
95
|
+
for index, mutation in inserts
|
|
96
|
+
if (refused := shape_refusal(contract, 0, index, mutation["entity"], mutation.get("value"))) is not None
|
|
97
|
+
]
|
|
98
|
+
if not refusals:
|
|
99
|
+
return
|
|
100
|
+
first = refusals[0]
|
|
101
|
+
summary = f"{len(refusals)} of {len(inserts)} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
|
|
102
|
+
context = {
|
|
103
|
+
"file": "data/initial.json",
|
|
104
|
+
"entity": first["entity"],
|
|
105
|
+
"index": first["index"],
|
|
106
|
+
"key": first["primaryKey"],
|
|
107
|
+
"field": first["field"],
|
|
108
|
+
"refused": len(refusals),
|
|
109
|
+
"rows": len(inserts),
|
|
110
|
+
"refusals": refusals,
|
|
111
|
+
}
|
|
112
|
+
row = initial["mutations"][first["index"]].get("value")
|
|
113
|
+
spec = contract.entities.get(first["entity"])
|
|
114
|
+
if spec is None:
|
|
115
|
+
fail("entity", summary + first["detail"], first["pointer"], **context)
|
|
116
|
+
if not isinstance(row, dict):
|
|
117
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
118
|
+
if any(name not in row for name in spec["primary_key"]):
|
|
119
|
+
fail("primary_key", summary + first["detail"], first["pointer"], **context)
|
|
120
|
+
try:
|
|
121
|
+
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
122
|
+
except ContractError as error:
|
|
123
|
+
raise error.contextualize(message=summary + error.detail["message"], **context)
|
|
124
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
125
|
+
|
|
126
|
+
|
|
83
127
|
def validate_schema_materialization(bundle: Path) -> None:
|
|
84
128
|
"""Validate compiled artifacts and initial state without retaining a database."""
|
|
85
129
|
bundle = Path(bundle)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""The one TOML reader for every Python the SDK supports.
|
|
2
|
+
|
|
3
|
+
3.11+ ships ``tomllib``; 3.10 gets ``tomli`` (declared in pyproject as
|
|
4
|
+
``tomli; python_version < '3.11'``). Import ``tomllib`` from here, never the
|
|
5
|
+
stdlib module directly, so the fallback lives in one place.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
import tomllib
|
|
10
|
+
except ModuleNotFoundError: # pragma: no cover - python 3.10
|
|
11
|
+
import tomli as tomllib # type: ignore[no-redef]
|
|
12
|
+
|
|
13
|
+
__all__ = ["tomllib"]
|
|
@@ -95,7 +95,6 @@ import sys
|
|
|
95
95
|
import tempfile
|
|
96
96
|
import threading
|
|
97
97
|
import time
|
|
98
|
-
import tomllib
|
|
99
98
|
import urllib.parse
|
|
100
99
|
import uuid
|
|
101
100
|
from dataclasses import dataclass, field
|
|
@@ -104,7 +103,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
104
103
|
from pathlib import Path
|
|
105
104
|
from typing import Any, Callable
|
|
106
105
|
|
|
106
|
+
from .._toml import tomllib
|
|
107
107
|
from . import snapshot as snapshots
|
|
108
|
+
from .bundle_runtime import stage_runtime_package
|
|
108
109
|
from .compiler import check_bundle, compile_bundle
|
|
109
110
|
from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
|
|
110
111
|
from .grading import keyed_entities
|
|
@@ -1973,12 +1974,7 @@ def build_api_bundle(world: str | Path, out: str | Path) -> Path:
|
|
|
1973
1974
|
f"The {spec.slug} world serving the vendor's API, seeded with its captured state. Point the provider client here.\n"
|
|
1974
1975
|
)
|
|
1975
1976
|
server = out_dir / "server"
|
|
1976
|
-
|
|
1977
|
-
package.mkdir(parents=True)
|
|
1978
|
-
(server / "gatewaysdk" / "__init__.py").write_text("")
|
|
1979
|
-
(server / "gatewaysdk" / "environment" / "__init__.py").write_text("")
|
|
1980
|
-
for module in sorted(Path(__file__).parent.glob("*.py")):
|
|
1981
|
-
shutil.copy(module, package / module.name)
|
|
1977
|
+
stage_runtime_package(server)
|
|
1982
1978
|
shutil.copytree(root, server / "world", ignore=shutil.ignore_patterns("captures", "__pycache__", ".git"))
|
|
1983
1979
|
(server / "main.py").write_text(
|
|
1984
1980
|
"\n".join(
|
|
@@ -355,8 +355,11 @@ def chunk_rows(
|
|
|
355
355
|
chunk_bytes: int = CHUNK_BYTES,
|
|
356
356
|
chunk_rows: int | None = None,
|
|
357
357
|
entity: str | None = None,
|
|
358
|
+
first_ordinal: int = 0,
|
|
358
359
|
) -> Iterator[Chunk]:
|
|
359
|
-
"""Cut a row stream into gzipped JSONL chunk files under ``workdir
|
|
360
|
+
"""Cut a row stream into gzipped JSONL chunk files under ``workdir``, numbering
|
|
361
|
+
them from ``first_ordinal`` (the ordinals below it belong to chunks the platform
|
|
362
|
+
staged from the project drive).
|
|
360
363
|
|
|
361
364
|
Lines are tagged ``{"entity", "row"}`` unless ``entity`` names the one entity
|
|
362
365
|
every row belongs to, in which case they are bare rows. A chunk closes when the
|
|
@@ -367,8 +370,10 @@ def chunk_rows(
|
|
|
367
370
|
workdir.mkdir(parents=True, exist_ok=True)
|
|
368
371
|
if chunk_bytes < 1 or (chunk_rows is not None and chunk_rows < 1):
|
|
369
372
|
fail("chunk", "chunk bounds must be positive", "/chunk")
|
|
373
|
+
if first_ordinal < 0:
|
|
374
|
+
fail("chunk", "the first ordinal must not be negative", "/chunk")
|
|
370
375
|
writer: _ChunkWriter | None = None
|
|
371
|
-
ordinal =
|
|
376
|
+
ordinal = first_ordinal
|
|
372
377
|
seen = 0
|
|
373
378
|
for name, row in rows:
|
|
374
379
|
if entity is not None and name != entity:
|
|
@@ -391,17 +396,18 @@ def chunk_rows(
|
|
|
391
396
|
|
|
392
397
|
|
|
393
398
|
def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tuple[str, dict]]:
|
|
394
|
-
"""Stream ``(entity, row)`` from one chunk file
|
|
395
|
-
|
|
399
|
+
"""Stream ``(entity, row)`` from one chunk file: gzip (sniffed, whatever the name)
|
|
400
|
+
or plain, holding any rows-file shape — the tagged or bare JSONL a CLI cut, or a
|
|
401
|
+
``{entity: [rows]}`` / bare-array document a project-drive file was staged as."""
|
|
396
402
|
path = Path(path)
|
|
397
403
|
with open(path, "rb") as probe:
|
|
398
404
|
compressed = probe.read(2) == b"\x1f\x8b"
|
|
399
405
|
opener = gzip.open if compressed else open
|
|
400
406
|
with opener(path, "rt", encoding="utf-8") as stream: # type: ignore[call-overload]
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
407
|
+
try:
|
|
408
|
+
yield from iter_file_rows(stream, str(path), entity=entity)
|
|
409
|
+
except UnicodeError:
|
|
410
|
+
fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
|
|
405
411
|
|
|
406
412
|
|
|
407
413
|
# ---------------------------------------------------------------- refusals
|
|
@@ -669,9 +675,7 @@ def _apply(
|
|
|
669
675
|
return applied, None
|
|
670
676
|
|
|
671
677
|
|
|
672
|
-
def validate_chunks(
|
|
673
|
-
contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None
|
|
674
|
-
) -> dict:
|
|
678
|
+
def validate_chunks(contract: Contract, chunks: list[ChunkSpec], *, redactor: "RowRedactor | None" = None) -> dict:
|
|
675
679
|
"""``validate-rows``: shape-only, every row of every chunk — after the batch's
|
|
676
680
|
redaction hook, when it has one, so a plaintext row under ``redact=refuse``
|
|
677
681
|
is refused here, in the validating phase, before anything is applied."""
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The runtime an api bundle carries beside its `server/main.py`: this package, vendored."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
SCHEMA_PACKAGE = Path(__file__).parent
|
|
9
|
+
ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
|
|
10
|
+
# Modules outside `schema/` that its modules import; copied beside them so `main.py` resolves them.
|
|
11
|
+
SIBLING_MODULES = ("_toml.py",)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def stage_runtime_package(server: Path) -> Path:
|
|
15
|
+
"""Vendor `gatewaysdk.environment.schema` (and what it imports) under `server/`; returns the package dir."""
|
|
16
|
+
package = server / "gatewaysdk" / "environment" / "schema"
|
|
17
|
+
package.mkdir(parents=True)
|
|
18
|
+
for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
|
|
19
|
+
(server / package_dir / "__init__.py").write_text("")
|
|
20
|
+
for name in SIBLING_MODULES:
|
|
21
|
+
shutil.copy(ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name)
|
|
22
|
+
for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
|
|
23
|
+
shutil.copy(module, package / module.name)
|
|
24
|
+
return package
|
|
@@ -8,11 +8,7 @@ from dataclasses import dataclass
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
import tomllib
|
|
13
|
-
except ModuleNotFoundError:
|
|
14
|
-
import tomli as tomllib
|
|
15
|
-
|
|
11
|
+
from .._toml import tomllib
|
|
16
12
|
from .validation import (
|
|
17
13
|
LANGUAGE,
|
|
18
14
|
ContractError,
|
|
@@ -51,6 +47,7 @@ FIELD_RESERVED = ("sqlite_", "gateway_", "benchmark_")
|
|
|
51
47
|
#: An entity may not take them.
|
|
52
48
|
RESERVED_STATE_KEYS = frozenset({"snapshot", "calls", "report"})
|
|
53
49
|
|
|
50
|
+
|
|
54
51
|
def identifier(value: Any, location: str) -> str:
|
|
55
52
|
if (
|
|
56
53
|
not isinstance(value, str)
|
|
@@ -632,6 +629,39 @@ def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
|
|
|
632
629
|
return contract
|
|
633
630
|
|
|
634
631
|
|
|
632
|
+
STALE_JOURNAL = (
|
|
633
|
+
"Batch targets a different domain identity/version/hash; a journal that holds "
|
|
634
|
+
"state is not migrated to an edited schema — re-import its rows against the new "
|
|
635
|
+
"contract (or restore the schema it was written against)"
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _journals_off_target(bundle: Path, contract: Contract) -> list[tuple[Path, str, dict]]:
|
|
640
|
+
"""Every journal whose target is not this contract's: ``(path, location, journal)``."""
|
|
641
|
+
from .validation import journal_paths, read_json
|
|
642
|
+
|
|
643
|
+
off = []
|
|
644
|
+
for path in journal_paths(bundle):
|
|
645
|
+
journal = read_json(path)
|
|
646
|
+
location = str(path.relative_to(bundle))
|
|
647
|
+
if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
|
|
648
|
+
fail("batch", "Journal must be a batch object with a target", location)
|
|
649
|
+
if journal["target"] != contract.target:
|
|
650
|
+
off.append((path, location, journal))
|
|
651
|
+
return off
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def stale_journals(bundle: str | Path, contract: Contract) -> list[str]:
|
|
655
|
+
"""The journals ``compile`` will refuse: rows written against another
|
|
656
|
+
contract. Read-only; what ``check`` names instead of \"run compile\" when
|
|
657
|
+
compile is the step that cannot succeed."""
|
|
658
|
+
return [
|
|
659
|
+
location
|
|
660
|
+
for _path, location, journal in _journals_off_target(Path(bundle), contract)
|
|
661
|
+
if journal.get("mutations") != []
|
|
662
|
+
]
|
|
663
|
+
|
|
664
|
+
|
|
635
665
|
def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]:
|
|
636
666
|
"""Re-stamp every journal that holds no state onto the compiled contract's
|
|
637
667
|
target, and refuse one that holds rows.
|
|
@@ -642,25 +672,11 @@ def retarget_empty_journals(bundle: str | Path, contract: Contract) -> list[str]
|
|
|
642
672
|
migrate in an empty journal, whatever domain it named, so it follows the
|
|
643
673
|
contract. A journal with rows is never migrated silently; the message names
|
|
644
674
|
the fix. Returns the bundle-relative paths rewritten."""
|
|
645
|
-
from .validation import journal_paths, read_json
|
|
646
|
-
|
|
647
675
|
bundle = Path(bundle)
|
|
648
676
|
retargeted = []
|
|
649
|
-
for path in
|
|
650
|
-
journal = read_json(path)
|
|
651
|
-
location = str(path.relative_to(bundle))
|
|
652
|
-
if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
|
|
653
|
-
fail("batch", "Journal must be a batch object with a target", location)
|
|
654
|
-
if journal["target"] == contract.target:
|
|
655
|
-
continue
|
|
677
|
+
for path, location, journal in _journals_off_target(bundle, contract):
|
|
656
678
|
if journal.get("mutations") != []:
|
|
657
|
-
fail(
|
|
658
|
-
"batch_target",
|
|
659
|
-
"Batch targets a different domain identity/version/hash; a journal that holds "
|
|
660
|
-
"state is not migrated to an edited schema — re-import its rows against the new "
|
|
661
|
-
"contract (or restore the schema it was written against)",
|
|
662
|
-
location,
|
|
663
|
-
)
|
|
679
|
+
fail("batch_target", STALE_JOURNAL, location)
|
|
664
680
|
journal["target"] = contract.target
|
|
665
681
|
path.write_bytes(encoded(journal))
|
|
666
682
|
retargeted.append(location)
|
|
@@ -690,18 +706,34 @@ def check_bundle(bundle: str | Path) -> Contract:
|
|
|
690
706
|
except OSError:
|
|
691
707
|
matches = False
|
|
692
708
|
if not matches:
|
|
693
|
-
fail(
|
|
694
|
-
"lock_mismatch",
|
|
695
|
-
"Missing, edited or incompatible contract artifact; the tree changed "
|
|
696
|
-
"since its last compile, so run `gateway worlds schema compile <dir>` "
|
|
697
|
-
"(state migration is not automatic)",
|
|
698
|
-
path,
|
|
699
|
-
)
|
|
709
|
+
fail("lock_mismatch", _stale_artifact_remedy(bundle, contract), path)
|
|
700
710
|
from .snapshot import check_all
|
|
701
711
|
|
|
702
712
|
check_all(bundle, contract)
|
|
703
713
|
return contract
|
|
704
714
|
|
|
705
715
|
|
|
716
|
+
def _stale_artifact_remedy(bundle: Path, contract: Contract) -> str:
|
|
717
|
+
"""What to do about a stale artifact. \"Run compile\" only when compile can
|
|
718
|
+
succeed; when a journal holds rows written against the previous contract,
|
|
719
|
+
compile refuses too, so name that step and its fix here (a check that sent
|
|
720
|
+
the author to a compile that failed the same way cost a dogfood session)."""
|
|
721
|
+
stale = stale_journals(bundle, contract)
|
|
722
|
+
if not stale:
|
|
723
|
+
return (
|
|
724
|
+
"Missing, edited or incompatible contract artifact; the tree changed "
|
|
725
|
+
"since its last compile, so run `gateway worlds schema compile <dir>` "
|
|
726
|
+
"(state migration is not automatic)"
|
|
727
|
+
)
|
|
728
|
+
return (
|
|
729
|
+
"Missing, edited or incompatible contract artifact; the schema changed since "
|
|
730
|
+
f"its last compile and {', '.join(stale)} holds rows written against the previous "
|
|
731
|
+
"contract, so `gateway worlds schema compile <dir>` will refuse too (batch_target). "
|
|
732
|
+
"Either restore the schema those rows were written against, or move the journal "
|
|
733
|
+
"aside, compile, and re-import the rows against the new contract "
|
|
734
|
+
"(`gateway worlds data import <dir> <rows.json>`)"
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
|
|
706
738
|
def describe_bundle(bundle: str | Path) -> str:
|
|
707
739
|
return build(Path(bundle)).artifacts["schema/README.md"].decode()
|
|
@@ -29,7 +29,6 @@ from __future__ import annotations
|
|
|
29
29
|
import base64
|
|
30
30
|
import json
|
|
31
31
|
import threading
|
|
32
|
-
import tomllib
|
|
33
32
|
import urllib.error
|
|
34
33
|
import urllib.parse
|
|
35
34
|
import urllib.request
|
|
@@ -37,6 +36,7 @@ from dataclasses import dataclass, field
|
|
|
37
36
|
from pathlib import Path
|
|
38
37
|
from typing import Any
|
|
39
38
|
|
|
39
|
+
from .._toml import tomllib
|
|
40
40
|
from .api import WorldState, load_api_settings, serve
|
|
41
41
|
from .connector import SPEC, Operation, load_spec
|
|
42
42
|
from .validation import fail
|
|
@@ -66,7 +66,6 @@ import re
|
|
|
66
66
|
import shutil
|
|
67
67
|
import tempfile
|
|
68
68
|
import time
|
|
69
|
-
import tomllib
|
|
70
69
|
import urllib.error
|
|
71
70
|
import urllib.parse
|
|
72
71
|
import urllib.request
|
|
@@ -75,6 +74,7 @@ from datetime import datetime, timezone
|
|
|
75
74
|
from pathlib import Path
|
|
76
75
|
from typing import Any, Callable
|
|
77
76
|
|
|
77
|
+
from .._toml import tomllib
|
|
78
78
|
from .compiler import check_bundle, compile_bundle
|
|
79
79
|
from .store import SQLiteStore
|
|
80
80
|
from .transforms import apply_transforms
|
package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Merge `google_drive` tool results onto this world's entities. Merge answers each
|
|
2
|
+
# tool with one JSON document; `list_files` carries the files under `files`, in
|
|
3
|
+
# snake_case (the Drive contract is camelCase, hence the renames).
|
|
4
|
+
|
|
5
|
+
[ingest]
|
|
6
|
+
on_conflict = "latest"
|
|
7
|
+
unmatched = "refuse"
|
|
8
|
+
|
|
9
|
+
[[sources]]
|
|
10
|
+
name = "list_files"
|
|
11
|
+
select = "result.files[*]"
|
|
12
|
+
entity = "drive_files"
|
|
13
|
+
at = "modified_time"
|
|
14
|
+
|
|
15
|
+
[sources.fields]
|
|
16
|
+
id = "id"
|
|
17
|
+
name = "name"
|
|
18
|
+
mimeType = "mime_type"
|
|
19
|
+
starred = "starred"
|
|
20
|
+
trashed = "trashed"
|
|
21
|
+
shared = "shared"
|
|
22
|
+
size = "size"
|
|
23
|
+
webViewLink = "web_view_link"
|
|
24
|
+
createdTime = "created_time"
|
|
25
|
+
modifiedTime = "modified_time"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# How the platform fills a world on this contract from a live Merge connection
|
|
2
|
+
# (`gateway worlds create --template google-drive --mode live --connection <id>`).
|
|
3
|
+
# Each [[pulls]] is one read tool of Merge's `google_drive` tool pack, called on
|
|
4
|
+
# the connection's registered user; readers/merge.ingest.toml maps what comes back
|
|
5
|
+
# onto the world's entities. Only read tools belong here.
|
|
6
|
+
|
|
7
|
+
[reader]
|
|
8
|
+
transport = "merge"
|
|
9
|
+
provider = "google_drive"
|
|
10
|
+
mapping = "merge.ingest.toml"
|
|
11
|
+
|
|
12
|
+
# Files the connected account can see; trashed ones stay out of the world.
|
|
13
|
+
[[pulls]]
|
|
14
|
+
tool = "list_files"
|
|
15
|
+
arguments = { page_size = 100, q = "trashed = false" }
|
|
16
|
+
page = { cursor = "page_token", next = "next_page_token" }
|
|
17
|
+
max_pages = 20
|
|
@@ -31,6 +31,7 @@ from dataclasses import dataclass, field
|
|
|
31
31
|
from pathlib import Path
|
|
32
32
|
from typing import IO, Any, Iterable, Iterator
|
|
33
33
|
|
|
34
|
+
from .._toml import tomllib
|
|
34
35
|
from .batch import _Reader, _stream_array, encode_row
|
|
35
36
|
from .compiler import Contract, build
|
|
36
37
|
from .connector import SPEC, Spec, redact
|
|
@@ -63,11 +64,6 @@ class Summary:
|
|
|
63
64
|
|
|
64
65
|
def envelope_key(world: Path) -> str:
|
|
65
66
|
"""``[api].envelope`` of the world's connector.toml; "" when unset or absent."""
|
|
66
|
-
try:
|
|
67
|
-
import tomllib
|
|
68
|
-
except ModuleNotFoundError: # pragma: no cover - python 3.10
|
|
69
|
-
import tomli as tomllib
|
|
70
|
-
|
|
71
67
|
spec = world / SPEC
|
|
72
68
|
if not spec.is_file():
|
|
73
69
|
return ""
|
|
@@ -35,11 +35,7 @@ from dataclasses import dataclass
|
|
|
35
35
|
from http.cookies import SimpleCookie
|
|
36
36
|
from pathlib import Path
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
import tomllib
|
|
40
|
-
except ModuleNotFoundError: # pragma: no cover - python < 3.11
|
|
41
|
-
import tomli as tomllib # type: ignore[no-redef]
|
|
42
|
-
|
|
38
|
+
from .._toml import tomllib
|
|
43
39
|
from .validation import fail
|
|
44
40
|
|
|
45
41
|
API_BUNDLE = "gateway-api-world.toml"
|
|
@@ -51,12 +51,12 @@ from __future__ import annotations
|
|
|
51
51
|
import hashlib
|
|
52
52
|
import json
|
|
53
53
|
import re
|
|
54
|
-
import tomllib
|
|
55
54
|
from dataclasses import dataclass
|
|
56
55
|
from datetime import datetime, timezone
|
|
57
56
|
from pathlib import Path
|
|
58
57
|
from typing import Any, Iterable
|
|
59
58
|
|
|
59
|
+
from .._toml import tomllib
|
|
60
60
|
from .transforms import apply_transforms, parse_transform
|
|
61
61
|
from .validation import ContractError, fail
|
|
62
62
|
|
|
@@ -326,7 +326,7 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
|
|
|
326
326
|
"domain": contract.target,
|
|
327
327
|
"package": contract.metadata["package"],
|
|
328
328
|
"initial_state": initial_state,
|
|
329
|
-
"handler_execution": _handler_execution(bundle, execute),
|
|
329
|
+
"handler_execution": _handler_execution(bundle, execute, declared),
|
|
330
330
|
"entity_counts": counts,
|
|
331
331
|
"tools": sorted(declared),
|
|
332
332
|
"verifiers": verifiers,
|
|
@@ -527,10 +527,14 @@ def may_execute() -> bool:
|
|
|
527
527
|
return os.environ.get(EXECUTE_ENV) == "1"
|
|
528
528
|
|
|
529
529
|
|
|
530
|
-
def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
|
|
530
|
+
def _handler_execution(bundle: Path, execute: bool, declared: set[str]) -> dict[str, Any]:
|
|
531
531
|
"""Whether the handlers were run: imported, every declared tool bound, the
|
|
532
532
|
``[dependencies]`` opened from their paths — one session of the world,
|
|
533
|
-
opened and closed. Only where authored code may run.
|
|
533
|
+
opened and closed. Only where authored code may run. ``tools`` is the
|
|
534
|
+
declared handlers the session bound (the report's top-level ``tools``);
|
|
535
|
+
``served`` every name ``worlds call`` answers there — handlers, the connector
|
|
536
|
+
routes as operations, a dependency's ``alias.tool`` — so the two lists differ
|
|
537
|
+
by design and say what each counts."""
|
|
534
538
|
if not execute:
|
|
535
539
|
return {
|
|
536
540
|
"status": "not_run",
|
|
@@ -546,7 +550,15 @@ def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
|
|
|
546
550
|
from .local import open_world
|
|
547
551
|
|
|
548
552
|
with open_world(bundle, http=False) as session:
|
|
549
|
-
|
|
553
|
+
served = list(session.tools)
|
|
554
|
+
unbound = sorted(declared - set(served))
|
|
555
|
+
if unbound:
|
|
556
|
+
fail(
|
|
557
|
+
"handler",
|
|
558
|
+
"Declared in schema/world.json (tools) but not bound when the session opened: " + ", ".join(unbound),
|
|
559
|
+
"/tools",
|
|
560
|
+
)
|
|
561
|
+
return {"status": "ok", "tools": sorted(declared), "served": served}
|
|
550
562
|
|
|
551
563
|
|
|
552
564
|
def world_root(root: Path) -> Path:
|
|
@@ -31,11 +31,11 @@ from __future__ import annotations
|
|
|
31
31
|
|
|
32
32
|
import json
|
|
33
33
|
import re
|
|
34
|
-
import tomllib
|
|
35
34
|
from dataclasses import dataclass, field
|
|
36
35
|
from pathlib import Path
|
|
37
36
|
from typing import Any, Callable
|
|
38
37
|
|
|
38
|
+
from .._toml import tomllib
|
|
39
39
|
from .validation import ContractError, fail
|
|
40
40
|
|
|
41
41
|
REDACT_MODES = ("off", "apply", "refuse")
|
|
@@ -81,6 +81,11 @@ TEMPLATE_CODE = {"handlers.py", "api/handlers.py"}
|
|
|
81
81
|
#: Python (``api/handlers.py``), its served scenarios (``scenarios/…``) and the
|
|
82
82
|
#: tests it ships for its tools (``tests/…``, run by ``gateway worlds test``).
|
|
83
83
|
TEMPLATE_DIRECTORIES = {"api", "scenarios", "tests", "ui"}
|
|
84
|
+
#: Template subdirectories that stay with the connector package and never enter a
|
|
85
|
+
#: world: authoring evidence, fidelity reports, and ``readers/`` — how the platform
|
|
86
|
+
#: fills a world on this contract from a live connection (``readers/merge.toml``
|
|
87
|
+
#: names the tools to pull, ``readers/merge.ingest.toml`` maps their payloads).
|
|
88
|
+
CONNECTOR_ONLY_DIRECTORIES = {"evidence", "reports", "readers"}
|
|
84
89
|
|
|
85
90
|
|
|
86
91
|
def _template_directory_files(directory: Path) -> list[Path]:
|
|
@@ -149,8 +154,8 @@ def init_bundle_from_files(
|
|
|
149
154
|
target = destination / "connector.toml"
|
|
150
155
|
elif path.startswith(("api/", "scenarios/", "tests/", "ui/")):
|
|
151
156
|
target = destination / path
|
|
152
|
-
elif path.
|
|
153
|
-
continue #
|
|
157
|
+
elif path.split("/", 1)[0] in CONNECTOR_ONLY_DIRECTORIES:
|
|
158
|
+
continue # evidence, reports and reader plans stay with the connector, not the world
|
|
154
159
|
elif (
|
|
155
160
|
Path(path).suffix in SUPPORTING_SUFFIXES and Path(path).name not in GENERATED_NAMES and "/" not in path
|
|
156
161
|
):
|
|
@@ -231,6 +236,8 @@ def init_bundle(
|
|
|
231
236
|
if not (path.name == "tests" and connector == CUSTOM and not with_example_tool):
|
|
232
237
|
code_files.extend(_template_directory_files(path))
|
|
233
238
|
continue
|
|
239
|
+
if path.name in CONNECTOR_ONLY_DIRECTORIES and path.is_dir() and not path.is_symlink():
|
|
240
|
+
continue
|
|
234
241
|
_resource(path)
|
|
235
242
|
if path.name in {"world.json", "handlers.py"}:
|
|
236
243
|
continue
|