@withgateway/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +3971 -2338
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3950 -2318
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +214 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +206 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
- package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1456 -564
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1433 -557
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +211 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +203 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +96 -21
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +96 -21
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +300 -123
- package/dist/worlds/index.d.ts +300 -123
- package/dist/worlds/index.js +1385 -508
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1377 -501
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -515
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +22 -14
- package/skills/connector-schema-authoring/connector-worlds.md +26 -10
- package/skills/connector-schema-authoring/world-schema.md +111 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +16 -16
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -41,6 +41,7 @@ from __future__ import annotations
|
|
|
41
41
|
import json
|
|
42
42
|
import shutil
|
|
43
43
|
import threading
|
|
44
|
+
import time
|
|
44
45
|
import uuid
|
|
45
46
|
from collections.abc import Callable
|
|
46
47
|
from dataclasses import dataclass, field
|
|
@@ -48,28 +49,29 @@ from pathlib import Path
|
|
|
48
49
|
from typing import Any
|
|
49
50
|
|
|
50
51
|
from .. import _bundle
|
|
51
|
-
from .._materialize import materialize_template
|
|
52
52
|
from .._toml import tomllib
|
|
53
53
|
from .._world import World
|
|
54
|
-
from ..errors import WorldStartError
|
|
54
|
+
from ..errors import VerifierError, WorldStartError
|
|
55
55
|
from ..world import World as WorldDetector
|
|
56
56
|
from . import snapshot as snapshots
|
|
57
57
|
from .api import ApiSettings, WorldState, load_api_settings
|
|
58
58
|
from .api import load_handlers as load_api_handlers
|
|
59
|
+
from .call_log import CallLog
|
|
59
60
|
from .compiler import ContractError, bundle_mode, check_bundle, fail
|
|
60
61
|
from .connector import SPEC, Spec, load_spec
|
|
61
62
|
from .data import seed_batch
|
|
63
|
+
from .database import WorldDatabase
|
|
62
64
|
from .dependencies import Dependency, parse_dependencies
|
|
63
65
|
from .grading import Grade, world_state
|
|
64
|
-
from .handler_errors import
|
|
66
|
+
from .handler_errors import failure_answer
|
|
67
|
+
from .served_names import ServedNames
|
|
68
|
+
from .session_files import ProvenTemplate, SessionExport
|
|
65
69
|
from .tasks import TaskSpec
|
|
66
70
|
|
|
67
71
|
Handler = Callable[..., Any]
|
|
68
72
|
|
|
69
73
|
#: The reward key the scalar goes under; a verifier may not claim it.
|
|
70
74
|
REWARD = "reward"
|
|
71
|
-
#: The opt-in table a bundle declares to read the active task from SQL.
|
|
72
|
-
TASK_CONTEXT_TABLE = "benchmark_runtime_context"
|
|
73
75
|
|
|
74
76
|
|
|
75
77
|
@dataclass(frozen=True)
|
|
@@ -119,7 +121,6 @@ class ToolsTask:
|
|
|
119
121
|
return list(self.worlds.get(alias, ()))
|
|
120
122
|
|
|
121
123
|
|
|
122
|
-
#: The id a session task takes when its spec names none.
|
|
123
124
|
INLINE_TASK_ID = "session"
|
|
124
125
|
|
|
125
126
|
|
|
@@ -138,14 +139,15 @@ def inline_task(spec: TaskSpec) -> ToolsTask:
|
|
|
138
139
|
@dataclass
|
|
139
140
|
class ToolsVersion:
|
|
140
141
|
"""A mock-tools version as the host caches it: the bundle, its tasks and
|
|
141
|
-
verifiers, the
|
|
142
|
-
|
|
142
|
+
verifiers, the proven template every session copies, and (in a worker)
|
|
143
|
+
the loaded handlers."""
|
|
143
144
|
|
|
144
145
|
bundle: Path
|
|
145
146
|
manifest: _bundle.EnvManifest
|
|
146
147
|
verifiers: list[str]
|
|
147
148
|
tasks: dict[str, ToolsTask]
|
|
148
|
-
template:
|
|
149
|
+
template: ProvenTemplate
|
|
150
|
+
engine: type[WorldDatabase]
|
|
149
151
|
#: The tools the version serves, by name: the loaded handlers' in a
|
|
150
152
|
#: worker, the contract's in the supervisor (which imports no handler but
|
|
151
153
|
#: still checks a session task's allow-list against what is served).
|
|
@@ -157,14 +159,12 @@ class ToolsVersion:
|
|
|
157
159
|
spec: Spec | None = None
|
|
158
160
|
settings: ApiSettings | None = None
|
|
159
161
|
api_handlers: dict[str, Handler] | None = None
|
|
160
|
-
#: The manifest's ``[dependencies]``, by alias; ``{}`` for a world of its own.
|
|
161
162
|
dependencies: dict[str, Dependency] = field(default_factory=dict)
|
|
162
163
|
|
|
163
164
|
@property
|
|
164
165
|
def served_names(self) -> list[str]:
|
|
165
|
-
"""Everything a session task may allow: the tools
|
|
166
|
-
|
|
167
|
-
return sorted(set(self.tool_names) | set(operations))
|
|
166
|
+
"""Everything a session task may allow: the declared tools."""
|
|
167
|
+
return sorted(self.tool_names)
|
|
168
168
|
|
|
169
169
|
|
|
170
170
|
@dataclass(frozen=True)
|
|
@@ -211,8 +211,8 @@ class DependencyWorld:
|
|
|
211
211
|
|
|
212
212
|
def served_with_dependencies(version: ToolsVersion, dependencies: dict[str, DependencyWorld]) -> list[str]:
|
|
213
213
|
"""Every name a session on ``version`` beside ``dependencies`` serves: the
|
|
214
|
-
root's tools
|
|
215
|
-
|
|
214
|
+
root's tools, and ``<alias>.<tool>`` for each dependency. Refuses a root
|
|
215
|
+
tool that reads as a dependency's (``slack.post_message``)."""
|
|
216
216
|
declared = set(version.dependencies)
|
|
217
217
|
missing = declared - set(dependencies)
|
|
218
218
|
if missing:
|
|
@@ -300,15 +300,13 @@ def scenario_tasks(bundle: Path) -> list[ToolsTask]:
|
|
|
300
300
|
return [_scenario_task(name, path) for name, path in snapshots.scenario_dirs(bundle).items()]
|
|
301
301
|
|
|
302
302
|
|
|
303
|
-
def load_tasks(
|
|
304
|
-
bundle: Path, manifest: _bundle.EnvManifest, verifiers: list[str], *, required: bool = True
|
|
305
|
-
) -> dict[str, ToolsTask]:
|
|
303
|
+
def load_tasks(bundle: Path, manifest: _bundle.EnvManifest, verifiers: list[str]) -> dict[str, ToolsTask]:
|
|
306
304
|
"""The bundle's tasks, in the runner's precedence: ``[[tasks]]`` in the
|
|
307
305
|
manifest, else ``tasks.jsonl`` / ``tasks.json``, else one task per
|
|
308
306
|
verifier; plus one per authored ``scenarios/<name>/`` the declared tasks
|
|
309
|
-
do not already name. A bundle
|
|
310
|
-
|
|
311
|
-
|
|
307
|
+
do not already name. A bundle may have none: a session that carries its
|
|
308
|
+
own task, or none (exploratory), still opens, and one that names a
|
|
309
|
+
bundled task the world lacks is refused when it opens."""
|
|
312
310
|
declared = manifest.raw.get("tasks")
|
|
313
311
|
tasks = _tasks_from_rows(declared) if isinstance(declared, list) else []
|
|
314
312
|
if not tasks:
|
|
@@ -341,11 +339,6 @@ def load_tasks(
|
|
|
341
339
|
by_id[task.id] = task
|
|
342
340
|
for task in scenario_tasks(bundle):
|
|
343
341
|
by_id.setdefault(task.id, task)
|
|
344
|
-
if not by_id and required:
|
|
345
|
-
fail(
|
|
346
|
-
"host",
|
|
347
|
-
"mock-tools bundle declares no tasks and ships no verifiers/*.sql: nothing to grade",
|
|
348
|
-
)
|
|
349
342
|
return by_id
|
|
350
343
|
|
|
351
344
|
|
|
@@ -355,10 +348,10 @@ def load_tools_version(
|
|
|
355
348
|
snapshot: dict[str, str] | None,
|
|
356
349
|
*,
|
|
357
350
|
run_handlers: bool,
|
|
358
|
-
require_tasks: bool = True,
|
|
359
351
|
) -> ToolsVersion:
|
|
360
|
-
"""Cache a mock-tools version: verify and pin its snapshot,
|
|
361
|
-
template once, read its tasks, and in a
|
|
352
|
+
"""Cache a mock-tools version: verify and pin its snapshot, build and prove
|
|
353
|
+
the template once (:class:`ProvenTemplate`), read its tasks, and in a
|
|
354
|
+
worker import its handlers."""
|
|
362
355
|
if not is_tools_bundle(bundle):
|
|
363
356
|
fail("host", f"{bundle} is not a mock-tools bundle (no gateway-env.toml)")
|
|
364
357
|
manifest = _bundle.parse_manifest(bundle)
|
|
@@ -373,12 +366,11 @@ def load_tools_version(
|
|
|
373
366
|
contract = check_bundle(bundle)
|
|
374
367
|
if snapshot:
|
|
375
368
|
snapshots.fetch(bundle, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
376
|
-
|
|
377
|
-
materialize_template(bundle, template, require_sql=True)
|
|
369
|
+
proven = ProvenTemplate.prove(bundle, template, contract, fetched=bool(snapshot))
|
|
378
370
|
verifiers = verifier_names(bundle)
|
|
379
371
|
if REWARD in verifiers:
|
|
380
372
|
fail("host", f"verifier name '{REWARD}' is reserved for the scalar grade")
|
|
381
|
-
tasks = load_tasks(bundle, manifest, verifiers
|
|
373
|
+
tasks = load_tasks(bundle, manifest, verifiers)
|
|
382
374
|
for task in tasks.values():
|
|
383
375
|
unknown = set(task.selected(verifiers)) - set(verifiers)
|
|
384
376
|
if unknown:
|
|
@@ -395,9 +387,7 @@ def load_tools_version(
|
|
|
395
387
|
handlers = _bundle.load_handlers(bundle, manifest)
|
|
396
388
|
except WorldStartError as error:
|
|
397
389
|
fail("host", str(error))
|
|
398
|
-
#
|
|
399
|
-
# api runtime reads it for a bundle, so a route that would not serve there
|
|
400
|
-
# is refused here, at cache time, not at the first request.
|
|
390
|
+
# connector.toml is read as the api runtime reads it, so an unservable route is refused at cache time.
|
|
401
391
|
spec = settings = api_handlers = None
|
|
402
392
|
if (bundle / SPEC).is_file():
|
|
403
393
|
spec = load_spec(bundle)
|
|
@@ -409,7 +399,8 @@ def load_tools_version(
|
|
|
409
399
|
manifest=manifest,
|
|
410
400
|
verifiers=verifiers,
|
|
411
401
|
tasks=tasks,
|
|
412
|
-
template=
|
|
402
|
+
template=proven,
|
|
403
|
+
engine=WorldDatabase.for_contract(contract),
|
|
413
404
|
tool_names=sorted(handlers) if handlers is not None else sorted(contract.tools),
|
|
414
405
|
handlers=handlers,
|
|
415
406
|
spec=spec,
|
|
@@ -420,13 +411,13 @@ def load_tools_version(
|
|
|
420
411
|
|
|
421
412
|
|
|
422
413
|
class _EnvHost:
|
|
423
|
-
"""What :class:`World` reads of an ``Environment
|
|
424
|
-
dir, the handlers, the bundle path, and always-``env``
|
|
414
|
+
"""What :class:`World` reads of an ``Environment`` beside the version's
|
|
415
|
+
template: a scratch dir, the handlers, the bundle path, and always-``env``
|
|
416
|
+
routing."""
|
|
425
417
|
|
|
426
418
|
def __init__(self, version: ToolsVersion, worlds_dir: Path) -> None:
|
|
427
419
|
self.bundle_path = version.bundle
|
|
428
420
|
self._worlds_dir = worlds_dir
|
|
429
|
-
self._template_db = version.template
|
|
430
421
|
self._handlers = version.handlers or {}
|
|
431
422
|
|
|
432
423
|
def resolve_route(self, tool_name: str) -> str:
|
|
@@ -449,7 +440,12 @@ class ToolsWorld:
|
|
|
449
440
|
``dependencies`` are the worlds the version's manifest declares, one
|
|
450
441
|
:class:`DependencyWorld` per alias, every one of them required: each gets
|
|
451
442
|
its own template copy under the session directory, and the root's store
|
|
452
|
-
reads each as ``<alias>.<table>``.
|
|
443
|
+
reads each as ``<alias>.<table>``.
|
|
444
|
+
|
|
445
|
+
``calls`` is the session's call log (its routes write it too when the tree
|
|
446
|
+
carries connector.toml): every tool call that runs is an entry
|
|
447
|
+
``via: tool``, so a grader over ``calls`` sees what the agent did through
|
|
448
|
+
tools."""
|
|
453
449
|
|
|
454
450
|
def __init__(
|
|
455
451
|
self,
|
|
@@ -460,22 +456,17 @@ class ToolsWorld:
|
|
|
460
456
|
seed: dict | None = None,
|
|
461
457
|
allowed: frozenset[str] | None = None,
|
|
462
458
|
dependencies: dict[str, DependencyWorld] | None = None,
|
|
459
|
+
calls: CallLog | None = None,
|
|
463
460
|
) -> None:
|
|
464
461
|
if version.handlers is None:
|
|
465
462
|
fail("host", "a mock-tools session runs world-authored handlers; only a version worker opens one")
|
|
466
463
|
for dep in (dependencies or {}).values():
|
|
467
464
|
if dep.version.handlers is None:
|
|
468
465
|
fail("host", f"dependency '{dep.alias}' has no handlers loaded; only a version worker opens one")
|
|
469
|
-
if isinstance(task, ToolsTask):
|
|
470
|
-
selected = task
|
|
471
|
-
else:
|
|
472
|
-
selected = version.tasks.get(task)
|
|
473
|
-
if selected is None:
|
|
474
|
-
fail("host", f"no task '{task}' in this world; it has {', '.join(sorted(version.tasks))}")
|
|
475
|
-
raise AssertionError # fail() never returns
|
|
476
466
|
self.version = version
|
|
477
|
-
self.task =
|
|
467
|
+
self.task = self._task_of(version, task)
|
|
478
468
|
self.allowed = allowed
|
|
469
|
+
self.calls = calls
|
|
479
470
|
self.dependencies = dict(dependencies or {})
|
|
480
471
|
self.served = served_with_dependencies(version, self.dependencies)
|
|
481
472
|
self._task_seed = seed
|
|
@@ -483,20 +474,34 @@ class ToolsWorld:
|
|
|
483
474
|
self._dir.mkdir(parents=True, exist_ok=True)
|
|
484
475
|
self.worlds: dict[str, World] = {}
|
|
485
476
|
try:
|
|
486
|
-
self.world =
|
|
477
|
+
self.world = self._open_world(version, self._dir)
|
|
487
478
|
for alias, dep in self.dependencies.items():
|
|
488
479
|
dep_dir = self._dir / "worlds" / alias
|
|
489
480
|
dep_dir.mkdir(parents=True, exist_ok=True)
|
|
490
|
-
self.worlds[alias] =
|
|
481
|
+
self.worlds[alias] = self._open_world(dep.version, dep_dir)
|
|
491
482
|
self._arm()
|
|
492
483
|
self._apply_dependency_scenarios()
|
|
493
484
|
self._apply_task_seed()
|
|
494
485
|
except BaseException:
|
|
495
|
-
# A seed the
|
|
496
|
-
# copy: the session directory goes with the world.
|
|
486
|
+
# A refused seed fails the open without leaking the session directory.
|
|
497
487
|
self.close()
|
|
498
488
|
raise
|
|
499
489
|
|
|
490
|
+
@staticmethod
|
|
491
|
+
def _task_of(version: ToolsVersion, task: str | ToolsTask) -> ToolsTask:
|
|
492
|
+
if isinstance(task, ToolsTask):
|
|
493
|
+
return task
|
|
494
|
+
selected = version.tasks.get(task)
|
|
495
|
+
if selected is None:
|
|
496
|
+
fail("host", f"no task '{task}' in this world; it has {', '.join(sorted(version.tasks)) or 'none'}")
|
|
497
|
+
raise AssertionError # fail() never returns
|
|
498
|
+
return selected
|
|
499
|
+
|
|
500
|
+
@staticmethod
|
|
501
|
+
def _open_world(version: ToolsVersion, directory: Path) -> World:
|
|
502
|
+
host = _EnvHost(version, directory)
|
|
503
|
+
return World(host, world_id=uuid.uuid4().hex, template=version.template) # type: ignore[arg-type]
|
|
504
|
+
|
|
500
505
|
def _apply_task_seed(self) -> None:
|
|
501
506
|
"""Start the session in its task's state: the template with the task's
|
|
502
507
|
rows on top — a scenario task's overlay replacing whole entities, then
|
|
@@ -520,17 +525,7 @@ class ToolsWorld:
|
|
|
520
525
|
attach each dependency's store to the root's for cross-world reads."""
|
|
521
526
|
scenario_id = str(self.task.metadata.get("scenario_id") or self.task.id)
|
|
522
527
|
for world in (self.world, *self.worlds.values()):
|
|
523
|
-
|
|
524
|
-
declared = conn.execute(
|
|
525
|
-
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (TASK_CONTEXT_TABLE,)
|
|
526
|
-
).fetchall()
|
|
527
|
-
if not declared:
|
|
528
|
-
continue
|
|
529
|
-
conn.execute(f"DELETE FROM {TASK_CONTEXT_TABLE}")
|
|
530
|
-
conn.execute(
|
|
531
|
-
f"INSERT INTO {TASK_CONTEXT_TABLE} (context_key, scenario_id) VALUES ('active_task', ?)",
|
|
532
|
-
(scenario_id,),
|
|
533
|
-
)
|
|
528
|
+
world.set_task_context(scenario_id)
|
|
534
529
|
self._attach()
|
|
535
530
|
|
|
536
531
|
def _attach(self) -> None:
|
|
@@ -546,7 +541,8 @@ class ToolsWorld:
|
|
|
546
541
|
|
|
547
542
|
def _route(self, tool: str) -> tuple[World, str]:
|
|
548
543
|
"""The world that serves ``tool`` and its name there: a dependency's by
|
|
549
|
-
``<alias>.<tool>``, else the root's.
|
|
544
|
+
``<alias>.<tool>``, else the root's. A root route's name is refused
|
|
545
|
+
with the api URL to call it over."""
|
|
550
546
|
alias, dot, local = tool.partition(".")
|
|
551
547
|
if dot and alias in self.dependencies:
|
|
552
548
|
dep = self.dependencies[alias]
|
|
@@ -554,44 +550,59 @@ class ToolsWorld:
|
|
|
554
550
|
return self.worlds[alias], local
|
|
555
551
|
elif tool in (self.version.handlers or {}):
|
|
556
552
|
return self.world, tool
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
553
|
+
self.names.refuse(tool)
|
|
554
|
+
|
|
555
|
+
@property
|
|
556
|
+
def names(self) -> ServedNames:
|
|
557
|
+
"""The tools this session serves, beside the root's routes it refuses by name."""
|
|
558
|
+
return ServedNames.of(self.served, self.version.spec)
|
|
560
559
|
|
|
561
560
|
def call(self, tool: str, args: dict) -> dict:
|
|
562
561
|
"""Run ``tool`` and answer as the handler's HTTP twin would: 200 with
|
|
563
562
|
the result, 400 when the handler itself is wrong (a name, attribute or
|
|
564
563
|
type error is the author's bug, never the engine's), 500 when anything
|
|
565
564
|
else fails inside the handler. Both failure bodies are structured
|
|
566
|
-
``{error, detail, handler}`` — never a bare exception string.
|
|
565
|
+
``{error, detail, handler}`` — never a bare exception string. A call
|
|
566
|
+
that runs is logged in :attr:`calls`, a contract refusal as its 400;
|
|
567
|
+
an unknown or disallowed name runs nothing and is not a call."""
|
|
567
568
|
world, local = self._route(tool)
|
|
568
569
|
if self.allowed is not None and tool not in self.allowed:
|
|
569
570
|
fail(
|
|
570
571
|
"tool_not_allowed",
|
|
571
572
|
f"tool '{tool}' is not allowed by this session's task; it allows: {', '.join(sorted(self.allowed))}",
|
|
572
573
|
)
|
|
574
|
+
started = time.monotonic()
|
|
575
|
+
status, body = 400, None
|
|
576
|
+
try:
|
|
577
|
+
status, body = self._run(world, local, args)
|
|
578
|
+
finally:
|
|
579
|
+
self._log(tool, args, status, body, time.monotonic() - started)
|
|
580
|
+
return {"status": status, "body": body}
|
|
581
|
+
|
|
582
|
+
@staticmethod
|
|
583
|
+
def _run(world: World, local: str, args: dict) -> tuple[int, Any]:
|
|
573
584
|
try:
|
|
574
|
-
|
|
585
|
+
return 200, world.execute(local, args)
|
|
575
586
|
except ContractError:
|
|
576
587
|
raise
|
|
577
|
-
except HANDLER_BUG_ERRORS as error:
|
|
578
|
-
return {"status": 400, "body": handler_bug_body(local, error)}
|
|
579
588
|
except Exception as error: # noqa: BLE001 - the handler's failure is the call's answer
|
|
580
|
-
return
|
|
581
|
-
|
|
589
|
+
return failure_answer(local, error)
|
|
590
|
+
|
|
591
|
+
def _log(self, tool: str, args: dict, status: int, body: Any, elapsed_s: float) -> None:
|
|
592
|
+
if self.calls is not None:
|
|
593
|
+
self.calls.record_tool(tool, args, status, body, elapsed_s)
|
|
582
594
|
|
|
583
595
|
def grade(self) -> dict:
|
|
584
596
|
"""The task's verifiers, the root's by name and each dependency's as
|
|
585
597
|
``<alias>/<name>``; ``reward`` is the primary verifier's score when the
|
|
586
598
|
task names one, else the mean of the numeric scores — null when no
|
|
587
599
|
verifier answered a number (never 0). A task no verifier grades is
|
|
588
|
-
ungraded, with the reason.
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
rewards
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
rewards[f"{alias}/{name}"] = self.worlds[alias].verify(name).score
|
|
600
|
+
ungraded, with the reason. A verifier that cannot score (bad SQL, a
|
|
601
|
+
value that is not a number from 0 to 1) refuses the grade by name."""
|
|
602
|
+
try:
|
|
603
|
+
rewards = self._verifier_scores()
|
|
604
|
+
except VerifierError as error:
|
|
605
|
+
fail("verifier", str(error))
|
|
595
606
|
if not rewards:
|
|
596
607
|
return Grade.ungraded().as_dict()
|
|
597
608
|
if self.task.verifier is not None and self.task.verifier in rewards:
|
|
@@ -601,6 +612,15 @@ class ToolsWorld:
|
|
|
601
612
|
reward = sum(numeric) / len(numeric) if numeric else None
|
|
602
613
|
return {REWARD: reward, "rewards": rewards}
|
|
603
614
|
|
|
615
|
+
def _verifier_scores(self) -> dict[str, float | None]:
|
|
616
|
+
rewards: dict[str, float | None] = {}
|
|
617
|
+
for name in self.task.selected(self.version.verifiers):
|
|
618
|
+
rewards[name] = self.world.verify(name).score
|
|
619
|
+
for alias, dep in self.dependencies.items():
|
|
620
|
+
for name in self.task.selected_of(alias, dep.version.verifiers):
|
|
621
|
+
rewards[f"{alias}/{name}"] = self.worlds[alias].verify(name).score
|
|
622
|
+
return rewards
|
|
623
|
+
|
|
604
624
|
def reset(self) -> dict:
|
|
605
625
|
for world in self.worlds.values():
|
|
606
626
|
world.reset()
|
|
@@ -611,16 +631,18 @@ class ToolsWorld:
|
|
|
611
631
|
return {"reset": True}
|
|
612
632
|
|
|
613
633
|
def export(self, destination: Path) -> Path:
|
|
614
|
-
"""The session's state as one
|
|
634
|
+
"""The session's state as one database file, taken consistently. A
|
|
615
635
|
session beside dependency worlds exports the root's state; theirs is
|
|
616
636
|
not in the file, so a restore starts them from their templates."""
|
|
617
637
|
return self.world.snapshot(destination)
|
|
618
638
|
|
|
619
|
-
def restore(self,
|
|
620
|
-
"""Continue from
|
|
639
|
+
def restore(self, export: SessionExport) -> None:
|
|
640
|
+
"""Continue from an export this host verified: the root's rows as the
|
|
641
|
+
export holds them, without a second row audit, and each dependency from
|
|
642
|
+
its template."""
|
|
621
643
|
for world in self.worlds.values():
|
|
622
644
|
world.reset()
|
|
623
|
-
self.world.
|
|
645
|
+
self.world.resume(export)
|
|
624
646
|
self._arm()
|
|
625
647
|
|
|
626
648
|
def _world_of(self, entity: str) -> tuple[World, str]:
|
|
@@ -655,9 +677,10 @@ class ToolsWorld:
|
|
|
655
677
|
batch = seed_batch(store, contract, local_rows, mode, batch_id=f"host-seed-{uuid.uuid4().hex}")
|
|
656
678
|
applied = store.apply_batch(batch)
|
|
657
679
|
seeded += applied.get("applied", sum(len(v) for v in local_rows.values()))
|
|
680
|
+
held = store.counts()
|
|
658
681
|
for entity in local_rows:
|
|
659
682
|
if entity in contract.entities:
|
|
660
|
-
counts[f"{prefix}.{entity}" if prefix else entity] =
|
|
683
|
+
counts[f"{prefix}.{entity}" if prefix else entity] = held[entity]
|
|
661
684
|
return {"seeded": seeded, "mode": mode, "entity_counts": counts}
|
|
662
685
|
|
|
663
686
|
@property
|
|
@@ -720,9 +743,9 @@ class ToolsWorldState(WorldState):
|
|
|
720
743
|
fail("host", "an http surface needs a schema-backed world (gateway-world/1); this bundle has none")
|
|
721
744
|
self.contract = contract
|
|
722
745
|
self._initial = None
|
|
723
|
-
self._snapshot_path = snapshots.default_snapshot(tree)
|
|
746
|
+
self._snapshot_path = snapshots.default_snapshot(tree, contract)
|
|
724
747
|
self.snapshot = None
|
|
725
|
-
self._lock = threading.
|
|
748
|
+
self._lock = threading.RLock()
|
|
726
749
|
self._clear_events()
|
|
727
750
|
|
|
728
751
|
@property
|
|
@@ -330,7 +330,7 @@ def _entity(name: str, entity: dict) -> dict:
|
|
|
330
330
|
return {
|
|
331
331
|
"title": title or _humanize(name),
|
|
332
332
|
"description": _short(schema.get("description")),
|
|
333
|
-
"primaryKey":
|
|
333
|
+
"primaryKey": [_row_field(key, set(properties)) or key for key in entity.get("primary_key") or []],
|
|
334
334
|
"fields": fields,
|
|
335
335
|
"relationships": relationships,
|
|
336
336
|
}
|
|
@@ -357,17 +357,34 @@ def _args(route: dict, entity: dict) -> dict[str, str] | None:
|
|
|
357
357
|
neither — never guessed at (a primary key is a field name like any other,
|
|
358
358
|
and is only used when the filter or the placeholder itself names it), so
|
|
359
359
|
such a route cannot address a row (or scope a list) of this entity and is
|
|
360
|
-
left for ``other`` instead.
|
|
360
|
+
left for ``other`` instead. A filter naming a nested field (``profile.email``)
|
|
361
|
+
fills it from a JSON Pointer (``/profile/email``), the way the page reads a
|
|
362
|
+
nested primary key."""
|
|
361
363
|
names = {f["name"] for f in entity["fields"]}
|
|
362
364
|
out: dict[str, str] = {}
|
|
363
365
|
for placeholder in route["placeholders"]:
|
|
364
|
-
field = route["filter"].get(placeholder, placeholder)
|
|
365
|
-
if field
|
|
366
|
+
field = _row_field(route["filter"].get(placeholder, placeholder), names)
|
|
367
|
+
if field is None:
|
|
366
368
|
return None
|
|
367
369
|
out[placeholder] = field
|
|
368
370
|
return out
|
|
369
371
|
|
|
370
372
|
|
|
373
|
+
def _row_field(path: str, names: set[str]) -> str | None:
|
|
374
|
+
"""A filter's row field as the page reads it: a top-level field by name, a nested
|
|
375
|
+
one under a top-level field as a JSON Pointer; None for anything else, an empty
|
|
376
|
+
step or a malformed pointer escape included."""
|
|
377
|
+
if path in names:
|
|
378
|
+
return path
|
|
379
|
+
pointer = [part.replace("~1", "/").replace("~0", "~") for part in path[1:].split("/")]
|
|
380
|
+
parts = pointer if path.startswith("/") else path.split(".")
|
|
381
|
+
if parts[0] not in names or "" in parts or re.search(r"~(?![01])", path if path.startswith("/") else ""):
|
|
382
|
+
return None
|
|
383
|
+
if len(parts) == 1:
|
|
384
|
+
return parts[0]
|
|
385
|
+
return "/" + "/".join(part.replace("~", "~0").replace("/", "~1") for part in parts)
|
|
386
|
+
|
|
387
|
+
|
|
371
388
|
def _classify(entity: dict, routes: list[dict]) -> tuple[dict, list[dict]]:
|
|
372
389
|
"""One route per verb, by method and shape — the declared action first, a
|
|
373
390
|
handler of the same shape as the fallback — and the rest under ``other``.
|
|
@@ -412,7 +429,9 @@ def _classify(entity: dict, routes: list[dict]) -> tuple[dict, list[dict]]:
|
|
|
412
429
|
unscoped_lists = [r for r in gets if not r["placeholders"] and not r["single"]]
|
|
413
430
|
scoped_lists = [r for r in placeholdered_gets if not addresses_row(r)]
|
|
414
431
|
pick("list", unscoped_lists + scoped_lists, read)
|
|
415
|
-
|
|
432
|
+
by_row = [r for r in placeholdered_gets if addresses_row(r)]
|
|
433
|
+
by_key = [r for r in by_row if set(args_of[id(r)].values()) <= set(entity["primaryKey"])]
|
|
434
|
+
pick("get", by_key or by_row, read)
|
|
416
435
|
pick(
|
|
417
436
|
"create",
|
|
418
437
|
[r for r in routes if r["method"] == "POST" and not r["placeholders"]],
|