@withgateway/sdk 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +13267 -5869
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +13272 -5874
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +83 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -28
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +4355 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/index.mjs +4243 -0
- package/dist/hooks/index.mjs.map +1 -0
- package/dist/{index-Dpo-t8iO.d.ts → index-1xnxt0Cc.d.ts} +58 -2
- package/dist/index-B0R3IgsV.d.mts +1279 -0
- package/dist/index-B0R3IgsV.d.ts +1279 -0
- package/dist/{index-B5HUQQDE.d.mts → index-C1wDEwH4.d.mts} +58 -2
- package/dist/index.d.mts +4 -3
- package/dist/index.d.ts +4 -3
- package/dist/index.js +6698 -1606
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6793 -1713
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +72 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +68 -17
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +47 -3
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +42 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +390 -27
- package/dist/worlds/index.d.ts +390 -27
- package/dist/worlds/index.js +2291 -366
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +2281 -356
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +7 -1
- package/runtime/gatewaysdk/environment/_bundle.py +14 -24
- package/runtime/gatewaysdk/environment/_materialize.py +3 -3
- package/runtime/gatewaysdk/environment/schema/api.py +153 -20
- package/runtime/gatewaysdk/environment/schema/compiler.py +14 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +171 -535
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +156 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +6 -5
- package/runtime/gatewaysdk/environment/schema/tasks.py +12 -2
- package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +44 -13
- package/runtime/gatewaysdk/environment/world.py +391 -0
- package/skills/hooks-getting-started/SKILL.md +156 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@withgateway/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5",
|
|
4
4
|
"description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -76,6 +76,11 @@
|
|
|
76
76
|
"import": "./dist/traces/index.mjs",
|
|
77
77
|
"require": "./dist/traces/index.js"
|
|
78
78
|
},
|
|
79
|
+
"./hooks": {
|
|
80
|
+
"types": "./dist/hooks/index.d.ts",
|
|
81
|
+
"import": "./dist/hooks/index.mjs",
|
|
82
|
+
"require": "./dist/hooks/index.js"
|
|
83
|
+
},
|
|
79
84
|
"./cli": {
|
|
80
85
|
"types": "./dist/cli.d.ts",
|
|
81
86
|
"import": "./dist/cli.mjs",
|
|
@@ -99,6 +104,7 @@
|
|
|
99
104
|
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
100
105
|
"test": "vitest",
|
|
101
106
|
"test:run": "vitest run",
|
|
107
|
+
"test:integration": "vitest run --config vitest.integration.config.ts",
|
|
102
108
|
"lint": "eslint src --ext .ts",
|
|
103
109
|
"clean": "rm -rf dist",
|
|
104
110
|
"example": "tsx examples/test-ab-experiment.ts",
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
"""Bundle inspection: kind detection, manifest parsing, handler loading.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- **harbor** — a tree of Harbor task dirs (``tasks/*/task.toml`` or
|
|
8
|
-
``*/task.toml``), mirroring ``benchmark_hub.harbor_adapter.detect_env_kind``.
|
|
3
|
+
Kind detection is :func:`gatewaysdk.environment.world.World.from_dir`; the
|
|
4
|
+
kinds ``gwenv`` runs in-process are **mock-tools** (``gateway-env.toml`` at the
|
|
5
|
+
bundle root, ``tools/*.py`` handlers, ``db/schema.sql``, ``verifiers/*.sql``)
|
|
6
|
+
and **harbor** (a tree of task dirs, ``tasks/*/task.toml`` or ``*/task.toml``).
|
|
9
7
|
|
|
10
8
|
Handler loading executes each ``tools/*.py`` with this SDK's
|
|
11
9
|
:mod:`gatewaysdk.environment.runtime` injected into ``sys.modules`` as
|
|
@@ -27,30 +25,22 @@ from typing import Any, Callable, Dict, List, Optional
|
|
|
27
25
|
|
|
28
26
|
from . import runtime
|
|
29
27
|
from .errors import EnvironmentNotReady, WorldStartError
|
|
30
|
-
|
|
31
|
-
KIND_MOCK_TOOLS = "mock-tools"
|
|
32
|
-
KIND_API = "api"
|
|
33
|
-
KIND_HARBOR = "harbor"
|
|
28
|
+
from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
|
|
34
29
|
|
|
35
30
|
|
|
36
31
|
def detect_kind(bundle_path: Path) -> str:
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
dirs (same glob heuristics as the runner's ``detect_env_kind``); else the
|
|
40
|
-
bundle is not runnable here."""
|
|
32
|
+
"""The tree's kind (``World.from_dir``), for a tree ``gwenv`` can run
|
|
33
|
+
in-process; a container benchmark package is not runnable here."""
|
|
41
34
|
from .schema import bundle_mode
|
|
42
35
|
|
|
43
36
|
bundle_mode(bundle_path) # Orphan/invalid schema declarations never downgrade.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
|
|
52
|
-
"a harbor task tree (no */task.toml) — nothing gwenv can run locally."
|
|
53
|
-
)
|
|
37
|
+
world = World.from_dir(bundle_path)
|
|
38
|
+
if not world.local_runtime:
|
|
39
|
+
raise EnvironmentNotReady(
|
|
40
|
+
f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
|
|
41
|
+
"a harbor task tree (no */task.toml) — nothing gwenv can run locally."
|
|
42
|
+
)
|
|
43
|
+
return world.kind
|
|
54
44
|
|
|
55
45
|
|
|
56
46
|
# ---------------------------------------------------------------------------
|
|
@@ -37,9 +37,9 @@ def materialize_template(bundle: Path, destination: Path, *, require_sql: bool =
|
|
|
37
37
|
|
|
38
38
|
raise ContractError(
|
|
39
39
|
"snapshot_pinned",
|
|
40
|
-
"db/snapshot.json pins a hosted snapshot that is not in the tree:
|
|
41
|
-
"
|
|
42
|
-
"
|
|
40
|
+
"db/snapshot.json pins a hosted snapshot that is not in the tree: run the world on "
|
|
41
|
+
"the platform (`gateway worlds session open <slug>`, load_world_to_sandbox), which "
|
|
42
|
+
"serves that snapshot, or remove db/snapshot.json to rebuild from the data files",
|
|
43
43
|
schema_path="/db/state.sqlite",
|
|
44
44
|
)
|
|
45
45
|
initial = read_json(bundle / "data/initial.json")
|
|
@@ -84,6 +84,7 @@ served beside the vendor routes.
|
|
|
84
84
|
from __future__ import annotations
|
|
85
85
|
|
|
86
86
|
import base64
|
|
87
|
+
import collections
|
|
87
88
|
import importlib.util
|
|
88
89
|
import json
|
|
89
90
|
import os
|
|
@@ -106,10 +107,24 @@ from typing import Any, Callable
|
|
|
106
107
|
from . import snapshot as snapshots
|
|
107
108
|
from .compiler import check_bundle, compile_bundle
|
|
108
109
|
from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
|
|
110
|
+
from .grading import keyed_entities
|
|
109
111
|
from .store import SQLiteStore
|
|
110
112
|
from .validation import ContractError, fail, read_json
|
|
111
113
|
|
|
112
|
-
ADMIN = {
|
|
114
|
+
ADMIN = {
|
|
115
|
+
"seed": "/__world/seed",
|
|
116
|
+
"reset": "/__world/reset",
|
|
117
|
+
"state": "/__world/state",
|
|
118
|
+
"calls": "/__world/calls",
|
|
119
|
+
"health": "/healthz",
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
#: How many vendor-route requests a world keeps in its call log, unless
|
|
123
|
+
#: ``WORLDS_CALL_LOG_MAX`` says otherwise; the oldest are dropped past it and
|
|
124
|
+
#: ``total`` still counts every one. ``0`` keeps none. The log is what a grader
|
|
125
|
+
#: reads to know what the agent ASKED (an investigation reads and never writes,
|
|
126
|
+
#: so its rows tell nothing) and what ``/__world/calls`` serves.
|
|
127
|
+
DEFAULT_CALL_LOG_MAX = 5000
|
|
113
128
|
#: How many events :meth:`WorldState.emit` keeps for a page polling ``since``.
|
|
114
129
|
EVENT_LOG_SIZE = 500
|
|
115
130
|
PORT = 8080
|
|
@@ -705,15 +720,12 @@ class WorldState:
|
|
|
705
720
|
def key_of(self, entity: str, row: dict) -> dict:
|
|
706
721
|
return {name: row.get(name) for name in self.contract.entities[entity]["primary_key"]}
|
|
707
722
|
|
|
723
|
+
def fields_of(self, entity: str) -> set[str]:
|
|
724
|
+
schema = self.contract.entities[entity].get("schema") or {}
|
|
725
|
+
return set((schema.get("properties") or {}).keys())
|
|
726
|
+
|
|
708
727
|
def dump(self) -> dict:
|
|
709
|
-
|
|
710
|
-
for entity, spec in self.contract.entities.items():
|
|
711
|
-
keyed = {}
|
|
712
|
-
for row in self.rows(entity):
|
|
713
|
-
key = ":".join(str(row.get(field)) for field in spec["primary_key"])
|
|
714
|
-
keyed[key] = row
|
|
715
|
-
out[entity] = keyed
|
|
716
|
-
return out
|
|
728
|
+
return keyed_entities(self.contract.entities, self.rows)
|
|
717
729
|
|
|
718
730
|
def describe_snapshot(self) -> dict | None:
|
|
719
731
|
"""What ``/__world/state`` reports beside the rows: the snapshot in use, or null."""
|
|
@@ -777,6 +789,17 @@ def _key_filter(operation: Operation, ctx: ApiContext, entity: str) -> dict[str,
|
|
|
777
789
|
return {name: name for name in operation.placeholders if name in keys}
|
|
778
790
|
|
|
779
791
|
|
|
792
|
+
def _query_field_filter(ctx: ApiContext, entity: str, sent: dict[str, list[str]]) -> dict[str, str]:
|
|
793
|
+
"""What a read route with no ``filter`` implies from its query string: each
|
|
794
|
+
parameter that names a top-level field of the entity narrows to rows carrying
|
|
795
|
+
that value (``GET /accounts?status=frozen``), the way every vendor's list
|
|
796
|
+
endpoint reads. Paging parameters and names the entity does not declare are
|
|
797
|
+
left to the pager and the handler."""
|
|
798
|
+
fields = ctx.fields_of(entity)
|
|
799
|
+
paging = {ctx.settings.paging.limit_param, ctx.settings.paging.offset_param, ctx.settings.paging.cursor_param}
|
|
800
|
+
return {name: name for name in sent if name in fields and name not in paging}
|
|
801
|
+
|
|
802
|
+
|
|
780
803
|
def _compare(have: str, wanted: str, mode: str) -> bool:
|
|
781
804
|
if mode == "contains":
|
|
782
805
|
return wanted in have
|
|
@@ -962,6 +985,9 @@ class ApiContext:
|
|
|
962
985
|
def key_of(self, entity: str, row: dict) -> dict:
|
|
963
986
|
return self.state.key_of(entity, row)
|
|
964
987
|
|
|
988
|
+
def fields_of(self, entity: str) -> set[str]:
|
|
989
|
+
return self.state.fields_of(entity)
|
|
990
|
+
|
|
965
991
|
# -- the live feed
|
|
966
992
|
def emit(self, event: dict) -> dict:
|
|
967
993
|
"""Record what this write changed for a page that polls (:meth:`WorldState.emit`)."""
|
|
@@ -1325,12 +1351,17 @@ def operation_request(spec: Spec, operation: Operation, args: dict) -> tuple[str
|
|
|
1325
1351
|
for key, value in path_params.items():
|
|
1326
1352
|
path = path.replace("{" + str(key) + "}", urllib.parse.quote(str(value), safe=""))
|
|
1327
1353
|
query = args.get("query") if isinstance(args.get("query"), dict) else {}
|
|
1328
|
-
if query:
|
|
1329
|
-
path = f"{path}?" + urllib.parse.urlencode(query)
|
|
1330
1354
|
body = args.get("body") if isinstance(args.get("body"), dict) else None
|
|
1331
|
-
if body
|
|
1332
|
-
|
|
1355
|
+
leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
|
|
1356
|
+
if operation.method == "GET" and leftovers:
|
|
1357
|
+
# A GET carries no body: a call written like a tool's (``list_accounts {"status": "frozen"}``)
|
|
1358
|
+
# means its query string, so the filter it asks for reaches the route.
|
|
1359
|
+
query = {**{k: v for k, v in leftovers.items() if not isinstance(v, (dict, list))}, **query}
|
|
1360
|
+
body = None
|
|
1361
|
+
elif body is None:
|
|
1333
1362
|
body = leftovers or None
|
|
1363
|
+
if query:
|
|
1364
|
+
path = f"{path}?" + urllib.parse.urlencode({k: str(v) for k, v in query.items()})
|
|
1334
1365
|
raw = json.dumps(body).encode("utf-8") if body is not None and operation.method != "GET" else b""
|
|
1335
1366
|
base = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
|
|
1336
1367
|
return operation.method, f"{base}{path}", raw
|
|
@@ -1342,6 +1373,95 @@ def operation_request(spec: Spec, operation: Operation, args: dict) -> tuple[str
|
|
|
1342
1373
|
Response = tuple[int, Any, dict[str, str] | None]
|
|
1343
1374
|
|
|
1344
1375
|
|
|
1376
|
+
def _strings_in(node: Any, out: list[str] | None = None) -> list[str]:
|
|
1377
|
+
"""Every string in a JSON value, however nested, once each, in document order."""
|
|
1378
|
+
if out is None:
|
|
1379
|
+
out = []
|
|
1380
|
+
if isinstance(node, str):
|
|
1381
|
+
if node not in out:
|
|
1382
|
+
out.append(node)
|
|
1383
|
+
elif isinstance(node, dict):
|
|
1384
|
+
for value in node.values():
|
|
1385
|
+
_strings_in(value, out)
|
|
1386
|
+
elif isinstance(node, (list, tuple)):
|
|
1387
|
+
for value in node:
|
|
1388
|
+
_strings_in(value, out)
|
|
1389
|
+
return out
|
|
1390
|
+
|
|
1391
|
+
|
|
1392
|
+
class CallLog:
|
|
1393
|
+
"""The vendor-route requests one session answered, in order: what the agent
|
|
1394
|
+
asked (operation, method, path, path args, query, body) and what came back
|
|
1395
|
+
(status, ``rows`` = the length of the operation's ``results`` list when the
|
|
1396
|
+
answer carries one, elapsed). Bounded; ``total`` counts past the bound.
|
|
1397
|
+
``/__world/calls`` serves it and a task's ``verify.py`` is handed it as
|
|
1398
|
+
``calls`` beside the rows, because a read-only investigation leaves no row
|
|
1399
|
+
behind and its calls are the only evidence of what it did."""
|
|
1400
|
+
|
|
1401
|
+
def __init__(self, maximum: int | None = None):
|
|
1402
|
+
if maximum is None:
|
|
1403
|
+
raw = os.environ.get("WORLDS_CALL_LOG_MAX")
|
|
1404
|
+
maximum = int(raw) if raw else DEFAULT_CALL_LOG_MAX
|
|
1405
|
+
self.maximum = max(0, int(maximum))
|
|
1406
|
+
self.entries: collections.deque[dict] = collections.deque(maxlen=self.maximum or None)
|
|
1407
|
+
self.total = 0
|
|
1408
|
+
self._lock = threading.Lock()
|
|
1409
|
+
|
|
1410
|
+
def record(
|
|
1411
|
+
self,
|
|
1412
|
+
operation: Operation,
|
|
1413
|
+
method: str,
|
|
1414
|
+
path: str,
|
|
1415
|
+
args: dict,
|
|
1416
|
+
query_all: dict,
|
|
1417
|
+
body: Any,
|
|
1418
|
+
status: int,
|
|
1419
|
+
answer: Any,
|
|
1420
|
+
elapsed_s: float,
|
|
1421
|
+
) -> None:
|
|
1422
|
+
if self.maximum == 0:
|
|
1423
|
+
return
|
|
1424
|
+
rows = None
|
|
1425
|
+
if isinstance(answer, dict) and isinstance(answer.get(operation.results), list):
|
|
1426
|
+
rows = len(answer[operation.results])
|
|
1427
|
+
elif isinstance(answer, list):
|
|
1428
|
+
rows = len(answer)
|
|
1429
|
+
query = {k: (v[0] if len(v) == 1 else list(v)) for k, v in query_all.items()}
|
|
1430
|
+
entry = {
|
|
1431
|
+
"seq": self.total + 1,
|
|
1432
|
+
"at": datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
|
|
1433
|
+
"operation": operation.name,
|
|
1434
|
+
"method": method,
|
|
1435
|
+
"path": path,
|
|
1436
|
+
"args": dict(args),
|
|
1437
|
+
"query": query,
|
|
1438
|
+
"body": body if isinstance(body, (dict, list)) and body else None,
|
|
1439
|
+
# Every string the request carried, wherever it sat (a path segment, a
|
|
1440
|
+
# query value, a body list of lists), so a grader can ask "was X searched"
|
|
1441
|
+
# with one `contains` and no knowledge of the vendor's request shape.
|
|
1442
|
+
"selectors": _strings_in([args, query, body]),
|
|
1443
|
+
"status": status,
|
|
1444
|
+
"rows": rows,
|
|
1445
|
+
# `found`: the request was answered and returned rows. A grader's `where`
|
|
1446
|
+
# takes scalars, and "asked and got something back" is the question an
|
|
1447
|
+
# investigation's grader asks most.
|
|
1448
|
+
"found": status < 400 and rows is not None and rows > 0,
|
|
1449
|
+
"elapsed_ms": round(elapsed_s * 1000, 1),
|
|
1450
|
+
}
|
|
1451
|
+
with self._lock:
|
|
1452
|
+
self.total += 1
|
|
1453
|
+
self.entries.append(entry)
|
|
1454
|
+
|
|
1455
|
+
def describe(self) -> dict:
|
|
1456
|
+
with self._lock:
|
|
1457
|
+
return {"total": self.total, "kept": len(self.entries), "max": self.maximum, "calls": list(self.entries)}
|
|
1458
|
+
|
|
1459
|
+
def clear(self) -> None:
|
|
1460
|
+
with self._lock:
|
|
1461
|
+
self.entries.clear()
|
|
1462
|
+
self.total = 0
|
|
1463
|
+
|
|
1464
|
+
|
|
1345
1465
|
class WorldApp:
|
|
1346
1466
|
"""One world's HTTP behaviour as a function of the request.
|
|
1347
1467
|
|
|
@@ -1359,6 +1479,7 @@ class WorldApp:
|
|
|
1359
1479
|
key: str | None,
|
|
1360
1480
|
handlers: dict[str, Handler] | None = None,
|
|
1361
1481
|
allowed: frozenset[str] | None = None,
|
|
1482
|
+
call_log_max: int | None = None,
|
|
1362
1483
|
):
|
|
1363
1484
|
self.state = state
|
|
1364
1485
|
self.router = router
|
|
@@ -1370,6 +1491,7 @@ class WorldApp:
|
|
|
1370
1491
|
#: A route outside the list answers 403 before any handler runs, so the
|
|
1371
1492
|
#: list holds against a client that talks to the vendor API directly.
|
|
1372
1493
|
self.allowed = allowed
|
|
1494
|
+
self.calls = CallLog(call_log_max)
|
|
1373
1495
|
|
|
1374
1496
|
# ------------------------------------------------------------ helpers
|
|
1375
1497
|
|
|
@@ -1431,7 +1553,9 @@ class WorldApp:
|
|
|
1431
1553
|
if url.path == ADMIN["health"]:
|
|
1432
1554
|
return 200, {"ok": True, "world": router.spec.slug}, None
|
|
1433
1555
|
if url.path == ADMIN["state"]:
|
|
1434
|
-
return 200, {**state.dump(), "snapshot": state.describe_snapshot()}, None
|
|
1556
|
+
return 200, {**state.dump(), "snapshot": state.describe_snapshot(), "calls": self.calls.describe()["calls"]}, None
|
|
1557
|
+
if url.path == ADMIN["calls"]:
|
|
1558
|
+
return 200, self.calls.describe(), None
|
|
1435
1559
|
if method == "POST":
|
|
1436
1560
|
if url.path == ADMIN["reset"]:
|
|
1437
1561
|
state.reset()
|
|
@@ -1477,6 +1601,13 @@ class WorldApp:
|
|
|
1477
1601
|
body = self._body(raw, headers) if method in ("POST", "PUT", "PATCH") else {}
|
|
1478
1602
|
if body is None:
|
|
1479
1603
|
return settings.invalid_status, settings.invalid, None
|
|
1604
|
+
started = time.monotonic()
|
|
1605
|
+
status, answer, extra = self._answer_route(op, operation, args, query, query_all, body, headers, url, default_host)
|
|
1606
|
+
self.calls.record(operation, method, path, args, query_all, body, status, answer, time.monotonic() - started)
|
|
1607
|
+
return status, answer, extra
|
|
1608
|
+
|
|
1609
|
+
def _answer_route(self, op, operation, args, query, query_all, body, headers, url, default_host) -> Response:
|
|
1610
|
+
settings, router, state, handlers = self.settings, self.router, self.state, self.handlers
|
|
1480
1611
|
host = headers.get("host") or default_host
|
|
1481
1612
|
forwarded = (headers.get("x-forwarded-proto") or "http").split(",")[0].strip()
|
|
1482
1613
|
ctx = ApiContext(state, router, operation, link=(f"{forwarded}://{host}{url.path}", query))
|
|
@@ -1513,17 +1644,19 @@ class WorldApp:
|
|
|
1513
1644
|
if not operation.entity:
|
|
1514
1645
|
return 501, {"error": "not_modelled", "operation": operation.name}
|
|
1515
1646
|
entity = operation.entity
|
|
1516
|
-
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
1517
|
-
# reads as "either", not as "a" with "b" silently dropped.
|
|
1518
|
-
values: dict[str, str | list[str]] = {
|
|
1519
|
-
**{k: v for k, v in ctx.query_all.items() if k in operation.params},
|
|
1520
|
-
**args,
|
|
1521
|
-
}
|
|
1522
1647
|
omit_done_cursor = ctx.settings.omits_done_cursor(op)
|
|
1523
1648
|
if op.action == "create":
|
|
1524
1649
|
row = ctx.insert(entity, _bind(op, ctx, body))
|
|
1525
1650
|
return op.status, row if op.single else wrap([row], ctx.settings, op.envelope)
|
|
1526
1651
|
filters = op.filter or _key_filter(operation, ctx, entity)
|
|
1652
|
+
if not op.filter and op.action == "read":
|
|
1653
|
+
filters = {**_query_field_filter(ctx, entity, ctx.query_all), **filters}
|
|
1654
|
+
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
1655
|
+
# reads as "either", not as "a" with "b" silently dropped.
|
|
1656
|
+
values: dict[str, str | list[str]] = {
|
|
1657
|
+
**{k: v for k, v in ctx.query_all.items() if k in operation.params or k in filters},
|
|
1658
|
+
**args,
|
|
1659
|
+
}
|
|
1527
1660
|
if op.action in ("update", "delete") and not filters:
|
|
1528
1661
|
fail(
|
|
1529
1662
|
"spec",
|
|
@@ -44,6 +44,13 @@ RESERVED = ("_", "sqlite_", "gateway_", "benchmark_")
|
|
|
44
44
|
FIELD_RESERVED = ("sqlite_", "gateway_", "benchmark_")
|
|
45
45
|
|
|
46
46
|
|
|
47
|
+
#: Keys a state dump carries beside its entities: ``snapshot`` (which snapshot the
|
|
48
|
+
#: rows came from), ``calls`` (the vendor-route requests the session answered, what
|
|
49
|
+
#: a grader reads when the rows say nothing) and ``report`` (the agent's own account
|
|
50
|
+
#: of the run, handed to the grade call, for a grader to check against the calls).
|
|
51
|
+
#: An entity may not take them.
|
|
52
|
+
RESERVED_STATE_KEYS = frozenset({"snapshot", "calls", "report"})
|
|
53
|
+
|
|
47
54
|
def identifier(value: Any, location: str) -> str:
|
|
48
55
|
if (
|
|
49
56
|
not isinstance(value, str)
|
|
@@ -291,6 +298,13 @@ def build(bundle: Path) -> Contract:
|
|
|
291
298
|
for name, entity in sorted(entities.items()):
|
|
292
299
|
location = f"world.json#/x-gateway/entities/{name}"
|
|
293
300
|
identifier(name, location)
|
|
301
|
+
if name.lower() in RESERVED_STATE_KEYS:
|
|
302
|
+
fail(
|
|
303
|
+
"identity",
|
|
304
|
+
f"'{name}' is a reserved key of the state dump (beside the rows it carries "
|
|
305
|
+
f"{', '.join(sorted(RESERVED_STATE_KEYS))}); name the entity something else",
|
|
306
|
+
location,
|
|
307
|
+
)
|
|
294
308
|
if name.lower() in seen:
|
|
295
309
|
fail("identity", "Entity names collide under SQLite case folding", location)
|
|
296
310
|
seen.add(name.lower())
|
|
@@ -35,7 +35,9 @@ from contextlib import contextmanager
|
|
|
35
35
|
from pathlib import Path
|
|
36
36
|
from typing import Any, Iterable, Iterator
|
|
37
37
|
|
|
38
|
+
from ..world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, SCHEMA_MANIFEST_KIND, World
|
|
38
39
|
from .compiler import bundle_mode, check_bundle, compile_bundle
|
|
40
|
+
from .proof import SCRATCH_IGNORE, RowProof
|
|
39
41
|
from .store import SQLiteStore
|
|
40
42
|
from .validation import ContractError, digest, encoded, fail, journal_paths, read_json, validate
|
|
41
43
|
|
|
@@ -48,14 +50,8 @@ INLINE_BYTES = 4 * 1024 * 1024
|
|
|
48
50
|
#: The most rows one ``entity_query`` page returns, the vendor pages' cap.
|
|
49
51
|
QUERY_LIMIT = 1000
|
|
50
52
|
QUERY_DEFAULT_LIMIT = 100
|
|
51
|
-
KIND_SCHEMA =
|
|
52
|
-
KIND_MOCK_TOOLS = "mock-tools"
|
|
53
|
-
KIND_API = "api"
|
|
54
|
-
KIND_HARBOR = "harbor"
|
|
55
|
-
KIND_UNKNOWN = "unknown"
|
|
53
|
+
KIND_SCHEMA = SCHEMA_MANIFEST_KIND
|
|
56
54
|
MAPPING = {"id": "data-import", "version": "1"}
|
|
57
|
-
#: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
|
|
58
|
-
SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
|
|
59
55
|
|
|
60
56
|
Rows = dict[str, list[dict]]
|
|
61
57
|
|
|
@@ -87,17 +83,13 @@ def parse_rows(paths: Iterable[str | Path], *, entity: str | None = None) -> Row
|
|
|
87
83
|
|
|
88
84
|
|
|
89
85
|
def world_kind(root: str | Path) -> str:
|
|
90
|
-
"""
|
|
86
|
+
"""The kind the worlds paths report (``World.from_dir(...).reported_kind``):
|
|
87
|
+
``schema``, ``mock-tools``, ``api``, ``harbor``, ``verifiers`` or
|
|
88
|
+
``verifiers-v1``. An invalid schema declaration refuses here rather than
|
|
89
|
+
downgrading the tree."""
|
|
91
90
|
root = Path(root)
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if (root / "gateway-api-world.toml").is_file():
|
|
95
|
-
return KIND_API
|
|
96
|
-
if (root / "gateway-env.toml").is_file():
|
|
97
|
-
return KIND_MOCK_TOOLS
|
|
98
|
-
if list(root.glob("tasks/*/task.toml")) or list(root.glob("*/task.toml")):
|
|
99
|
-
return KIND_HARBOR
|
|
100
|
-
return KIND_UNKNOWN
|
|
91
|
+
bundle_mode(root)
|
|
92
|
+
return World.from_dir(root).reported_kind
|
|
101
93
|
|
|
102
94
|
|
|
103
95
|
def _refuse_kind(kind: str, root: Path) -> None:
|
|
@@ -135,6 +127,7 @@ def import_rows(
|
|
|
135
127
|
merge: str = "upsert",
|
|
136
128
|
batch_id: str | None = None,
|
|
137
129
|
redact: str = "off",
|
|
130
|
+
dry_run: bool = False,
|
|
138
131
|
) -> dict:
|
|
139
132
|
"""Write ``rows`` into the world at ``root`` the way its kind seeds.
|
|
140
133
|
|
|
@@ -150,6 +143,13 @@ def import_rows(
|
|
|
150
143
|
``redact`` — ``off`` (default) stores the rows as given; ``apply`` runs them
|
|
151
144
|
through the world's ``[redact]`` policy first; ``refuse`` rejects a row that
|
|
152
145
|
carries plaintext in a redacted field (:mod:`.redaction`).
|
|
146
|
+
|
|
147
|
+
``dry_run`` proves the rows and writes nothing (``changed`` always ``False``):
|
|
148
|
+
the :class:`.proof.RowProof` the tree calls for — the same gate over a scratch
|
|
149
|
+
copy of an authored tree, the platform's batch lane over the hosted snapshot a
|
|
150
|
+
pinned checkout carries, or shape-only with references and unique keys
|
|
151
|
+
``deferred`` to the platform when the pinned checkout lacks its snapshot. A
|
|
152
|
+
pinned checkout refuses a real import: the rows belong on the platform.
|
|
153
153
|
"""
|
|
154
154
|
from .redaction import parse_mode, redact_rows, require_policy
|
|
155
155
|
|
|
@@ -160,6 +160,8 @@ def import_rows(
|
|
|
160
160
|
fail("merge", f"merge must be one of {', '.join(MERGES)}", "/merge")
|
|
161
161
|
if not isinstance(rows, dict) or not all(isinstance(v, list) for v in rows.values()):
|
|
162
162
|
fail("rows", "rows must map entity names to arrays of rows", "/rows")
|
|
163
|
+
if dry_run:
|
|
164
|
+
return RowProof.of(root).run(rows, mode=mode, merge=merge, batch_id=batch_id, redact=redact)
|
|
163
165
|
redact = parse_mode(redact)
|
|
164
166
|
rows = redact_rows(require_policy(root, redact), rows, redact)
|
|
165
167
|
kind = world_kind(root)
|
|
@@ -186,7 +188,7 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
|
|
|
186
188
|
reports."""
|
|
187
189
|
from .batch import shape_refusal
|
|
188
190
|
|
|
189
|
-
contract = compile_bundle(root)
|
|
191
|
+
contract = compile_bundle(root, snapshots=False)
|
|
190
192
|
refusals = [
|
|
191
193
|
refused
|
|
192
194
|
for entity, listed in rows.items()
|
|
@@ -220,7 +222,7 @@ def _merged(root: Path, rows: Rows, merge: str) -> Rows:
|
|
|
220
222
|
plan will refuse (not an object, key missing) is left in place for it."""
|
|
221
223
|
if merge != "fill":
|
|
222
224
|
return rows
|
|
223
|
-
contract = compile_bundle(root)
|
|
225
|
+
contract = compile_bundle(root, snapshots=False)
|
|
224
226
|
settled: Rows = {}
|
|
225
227
|
for entity, listed in rows.items():
|
|
226
228
|
first: dict[tuple, dict] = {}
|