@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
"""The one engine a session runs on. :class:`WorldRuntime` is what the World
|
|
2
|
-
Host holds for a session opened in its process: the store, the task's
|
|
3
|
-
starting state, the HTTP surface when the tree declares one, and every
|
|
4
|
-
control verb (reset, seed, state, grade, call, export, restore, close). The
|
|
5
|
-
host asks the runtime; it never asks what shape the version was stored as.
|
|
6
|
-
|
|
7
|
-
Two runtimes exist because two stored shapes still do:
|
|
8
|
-
|
|
9
|
-
- :class:`TreeRuntime` — a schema world stored as itself (``gateway-env.toml``
|
|
10
|
-
with a ``[schema]`` table): a copy of the version's proven template,
|
|
11
|
-
executed through its handlers, beside a copy of each dependency world, its
|
|
12
|
-
routes (``connector.toml``) over the same store. This is the runtime every
|
|
13
|
-
new push opens on.
|
|
14
|
-
- :class:`SnapshotRuntime` — an api bundle (``gateway-api-world.toml`` over
|
|
15
|
-
``server/world``): a clone of the version's compiled snapshot behind its
|
|
16
|
-
routes. **READ PATH, marked for deletion**: it exists so the worlds the
|
|
17
|
-
platform already stores as bundles keep opening until the migration
|
|
18
|
-
(handoff step 5) has rewritten them as schema worlds; nothing new is
|
|
19
|
-
stored in this shape. Delete it, ``CachedVersion.snapshot`` and the
|
|
20
|
-
api-bundle arm of ``VersionCache._build`` together (step 6).
|
|
21
|
-
|
|
22
|
-
The shape is chosen once, in :meth:`WorldRuntime.open`, from what the version
|
|
23
|
-
cache loaded; from there the host sees one interface."""
|
|
24
|
-
|
|
25
1
|
from __future__ import annotations
|
|
26
2
|
|
|
27
3
|
import json
|
|
@@ -33,37 +9,36 @@ from pathlib import Path
|
|
|
33
9
|
from typing import TYPE_CHECKING, Any, Optional
|
|
34
10
|
|
|
35
11
|
from . import snapshot as snapshots
|
|
36
|
-
from .api import WorldApp, WorldState
|
|
37
|
-
from .call_log import CallLog
|
|
12
|
+
from .api import WorldApp, WorldState
|
|
13
|
+
from .call_log import CallJournal, CallLog
|
|
38
14
|
from .database import WorldDatabase
|
|
39
15
|
from .dependencies import Dependency
|
|
40
16
|
from .grading import Grade, GraderError, TaskGrader
|
|
17
|
+
from .journal import CallsView, WorldJournal
|
|
18
|
+
from .request_ledger import RequestOutbox
|
|
41
19
|
from .served_names import ServedNames
|
|
42
|
-
from .session_files import SessionExport
|
|
20
|
+
from .session_files import CALLS_JOURNAL, LEDGER_JOURNAL, SessionExport
|
|
43
21
|
from .snapshot import SCENARIOS
|
|
44
|
-
from .tasks import
|
|
22
|
+
from .tasks import (
|
|
23
|
+
TASK_DIR,
|
|
24
|
+
TaskSpec,
|
|
25
|
+
_parse_grader,
|
|
26
|
+
parse_task_spec,
|
|
27
|
+
unknown_task_entities,
|
|
28
|
+
verifier_source,
|
|
29
|
+
write_task_dir,
|
|
30
|
+
)
|
|
45
31
|
from .tools_world import DependencyWorld, ToolsWorld, ToolsWorldState, inline_task, served_with_dependencies
|
|
46
32
|
from .validation import fail
|
|
47
33
|
from .workers import ConfinedVerifierSandbox
|
|
48
34
|
|
|
49
35
|
if TYPE_CHECKING:
|
|
50
|
-
from .
|
|
51
|
-
|
|
52
|
-
# The router still authorizes by it, so the host's own calls carry it.
|
|
53
|
-
WORLD_AUTH_SENTINEL = "gateway-session"
|
|
36
|
+
from .host_versions import CachedVersion
|
|
54
37
|
|
|
55
38
|
Restorer = Callable[[Callable[[SessionExport], None]], bool]
|
|
56
39
|
|
|
57
40
|
|
|
58
41
|
class WorldRuntime(ABC):
|
|
59
|
-
"""One session's world, in process. Built by :meth:`open`; the host keeps
|
|
60
|
-
it on the session and calls nothing else of the world's. ``app`` is the
|
|
61
|
-
routes of connector.toml over the session's store, None for a world of
|
|
62
|
-
tools alone; ``calls`` is the one call log its routes and its tools write;
|
|
63
|
-
``grant`` is an ``oauth2`` world's client-credentials
|
|
64
|
-
grant the host minted for itself, so a caller the session token admits is
|
|
65
|
-
presented to the world as a granted client."""
|
|
66
|
-
|
|
67
42
|
def __init__(
|
|
68
43
|
self,
|
|
69
44
|
cached: CachedVersion,
|
|
@@ -72,6 +47,7 @@ class WorldRuntime(ABC):
|
|
|
72
47
|
task: str | None,
|
|
73
48
|
spec: TaskSpec | None,
|
|
74
49
|
token: str | None,
|
|
50
|
+
journaled: bool = False,
|
|
75
51
|
) -> None:
|
|
76
52
|
self.cached = cached
|
|
77
53
|
self.session_dir = Path(session_dir)
|
|
@@ -79,24 +55,31 @@ class WorldRuntime(ABC):
|
|
|
79
55
|
self.task_spec = spec
|
|
80
56
|
self.token = token
|
|
81
57
|
self.app: WorldApp | None = None
|
|
82
|
-
|
|
83
|
-
self.
|
|
58
|
+
journal = CallJournal(self.session_dir / CALLS_JOURNAL) if journaled else None
|
|
59
|
+
outbox = RequestOutbox(CallJournal(self.session_dir / LEDGER_JOURNAL) if journaled else None)
|
|
60
|
+
spec = cached.spec
|
|
61
|
+
self.calls = CallLog.for_spec(spec, journal, outbox, token)
|
|
84
62
|
self.restored = False
|
|
85
63
|
|
|
86
64
|
@classmethod
|
|
87
65
|
@abstractmethod
|
|
88
|
-
def served_by(cls, cached: CachedVersion, dependencies: dict[str, DependencyWorld]) -> list[str]:
|
|
89
|
-
"""Every tool a session task on ``cached`` may allow."""
|
|
66
|
+
def served_by(cls, cached: CachedVersion, dependencies: dict[str, DependencyWorld]) -> list[str]: ...
|
|
90
67
|
|
|
91
68
|
@classmethod
|
|
92
69
|
@abstractmethod
|
|
93
70
|
def authored_code(cls, cached: CachedVersion) -> str | None:
|
|
94
|
-
"""
|
|
95
|
-
it), None when it runs none."""
|
|
71
|
+
"""World-authored code ``cached`` runs, so only a version worker may open it; None when it runs none."""
|
|
96
72
|
|
|
97
73
|
@classmethod
|
|
98
74
|
def declared_dependencies(cls, cached: CachedVersion) -> dict[str, Dependency]:
|
|
99
|
-
|
|
75
|
+
return {}
|
|
76
|
+
|
|
77
|
+
@staticmethod
|
|
78
|
+
@abstractmethod
|
|
79
|
+
def live_state(session_dir: Path) -> Path: ...
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def dependency_states(session_dir: Path) -> dict[str, Path]:
|
|
100
83
|
return {}
|
|
101
84
|
|
|
102
85
|
@classmethod
|
|
@@ -110,11 +93,10 @@ class WorldRuntime(ABC):
|
|
|
110
93
|
token: str | None,
|
|
111
94
|
dependencies: dict[str, DependencyWorld],
|
|
112
95
|
restore: Restorer,
|
|
96
|
+
journaled: bool = False,
|
|
113
97
|
) -> WorldRuntime:
|
|
114
|
-
"""
|
|
115
|
-
|
|
116
|
-
attached. A start that fails leaves no session directory behind."""
|
|
117
|
-
runtime = cached.runtime_class(cached, session_dir, task=task, spec=spec, token=token)
|
|
98
|
+
"""``journaled`` keeps the calls on disk, so a version worker that dies leaves them to the next one."""
|
|
99
|
+
runtime = cached.runtime_class(cached, session_dir, task=task, spec=spec, token=token, journaled=journaled)
|
|
118
100
|
runtime._start(dependencies)
|
|
119
101
|
try:
|
|
120
102
|
if spec is not None:
|
|
@@ -141,7 +123,6 @@ class WorldRuntime(ABC):
|
|
|
141
123
|
routes_closed=self.allowed is not None,
|
|
142
124
|
calls=self.calls,
|
|
143
125
|
)
|
|
144
|
-
self.grant = client_grant(self.app, self.token)
|
|
145
126
|
|
|
146
127
|
@property
|
|
147
128
|
def allowed(self) -> frozenset[str] | None:
|
|
@@ -149,10 +130,7 @@ class WorldRuntime(ABC):
|
|
|
149
130
|
return frozenset(spec.tools) if spec is not None and spec.tools is not None else None
|
|
150
131
|
|
|
151
132
|
def _check_task_entities(self, spec: TaskSpec) -> None:
|
|
152
|
-
"""A
|
|
153
|
-
names. The seed would fail on its own; an assertion over an unknown
|
|
154
|
-
entity would not, selecting no rows, so ``absent`` or ``count: eq 0``
|
|
155
|
-
checks would pass on a typo and grade a task that can never be graded."""
|
|
133
|
+
"""A check over an unknown entity selects no rows: ``absent`` or ``count: eq 0`` would pass on a typo."""
|
|
156
134
|
defined = self.entity_names()
|
|
157
135
|
unknown = unknown_task_entities(spec, defined)
|
|
158
136
|
if unknown:
|
|
@@ -165,22 +143,19 @@ class WorldRuntime(ABC):
|
|
|
165
143
|
|
|
166
144
|
@abstractmethod
|
|
167
145
|
def _start(self, dependencies: dict[str, DependencyWorld]) -> None:
|
|
168
|
-
"""
|
|
146
|
+
"""A failure here leaves nothing to close: :meth:`open` runs it outside its cleanup."""
|
|
169
147
|
|
|
170
148
|
def _to_start(self) -> None:
|
|
171
149
|
"""Lay the task's starting state on the store, when :meth:`_start` did not."""
|
|
172
150
|
|
|
173
151
|
@abstractmethod
|
|
174
|
-
def _app_state(self) -> Any:
|
|
175
|
-
"""The :class:`api.WorldState` the routes read and write."""
|
|
152
|
+
def _app_state(self) -> Any: ...
|
|
176
153
|
|
|
177
154
|
@abstractmethod
|
|
178
|
-
def entity_names(self) -> list[str]:
|
|
179
|
-
"""Every entity a grader may name (a dependency's as ``alias.entity``)."""
|
|
155
|
+
def entity_names(self) -> list[str]: ...
|
|
180
156
|
|
|
181
157
|
@abstractmethod
|
|
182
|
-
def served(self) -> list[str]:
|
|
183
|
-
"""Every name :meth:`call` answers: the declared tools, a dependency's as ``alias.tool``."""
|
|
158
|
+
def served(self) -> list[str]: ...
|
|
184
159
|
|
|
185
160
|
@abstractmethod
|
|
186
161
|
def reset(self) -> dict: ...
|
|
@@ -189,85 +164,79 @@ class WorldRuntime(ABC):
|
|
|
189
164
|
def seed(self, rows: dict, mode: str, redact: str) -> dict: ...
|
|
190
165
|
|
|
191
166
|
@abstractmethod
|
|
192
|
-
def _rows(self) -> dict:
|
|
193
|
-
"""The state without the calls: rows by entity, and what the store started from."""
|
|
167
|
+
def _rows(self) -> dict: ...
|
|
194
168
|
|
|
195
169
|
@abstractmethod
|
|
196
|
-
def _grader_rows(self) -> dict:
|
|
197
|
-
"""The rows as a session task's grader reads them (keyed by primary key)."""
|
|
170
|
+
def _grader_rows(self) -> dict: ...
|
|
198
171
|
|
|
199
172
|
@abstractmethod
|
|
200
|
-
def bundled_grade(self) -> dict | None:
|
|
201
|
-
"""
|
|
202
|
-
(a schema world's SQL verifiers), or None when the version grades a
|
|
203
|
-
bundled task through a ``verify.py`` the :class:`TaskGrader` runs."""
|
|
173
|
+
def bundled_grade(self, journal: WorldJournal | None = None) -> dict | None:
|
|
174
|
+
"""None means the bundled task grades by its ``verify.py`` (:class:`TaskGrader`), not that it is ungraded."""
|
|
204
175
|
|
|
205
176
|
@abstractmethod
|
|
206
|
-
def _bundled_verifier(self) -> Path | None:
|
|
207
|
-
"""The ``verify.py`` of the bundled task, when the version carries one."""
|
|
177
|
+
def _bundled_verifier(self) -> Path | None: ...
|
|
208
178
|
|
|
209
179
|
@abstractmethod
|
|
210
180
|
def call(self, tool: str, args: dict) -> dict:
|
|
211
|
-
"""A
|
|
212
|
-
refused with the api URL to call it over: routes are never tools."""
|
|
181
|
+
"""A route's name is refused with the api URL to call it over: routes are never tools."""
|
|
213
182
|
|
|
214
183
|
@abstractmethod
|
|
215
|
-
def _export_state(self, destination: Path) -> Path:
|
|
216
|
-
"""The store's copy of the session's state at ``destination``."""
|
|
184
|
+
def _export_state(self, destination: Path) -> Path: ...
|
|
217
185
|
|
|
218
186
|
@abstractmethod
|
|
219
187
|
def _restore_state(self, export: SessionExport) -> None:
|
|
220
188
|
"""The session's store laid from a verified export, without a row audit."""
|
|
221
189
|
|
|
222
190
|
def export(self, destination: Path) -> Path:
|
|
223
|
-
"""The session's state as one database file, with its call log in it, so a
|
|
224
|
-
reopen resumes both (:class:`SessionExport`)."""
|
|
225
191
|
path = self._export_state(destination)
|
|
226
192
|
SessionExport.carry_calls(path, self.calls.describe()["calls"], self.cached.engine)
|
|
227
193
|
return path
|
|
228
194
|
|
|
229
195
|
def restore(self, export: SessionExport) -> None:
|
|
230
|
-
"""Continue from an export this host verified, its call log with it."""
|
|
231
196
|
self._restore_state(export)
|
|
232
197
|
self.calls.resume(export.calls)
|
|
198
|
+
self.calls.outbox.resume(export.unsent)
|
|
233
199
|
|
|
234
200
|
@abstractmethod
|
|
235
201
|
def close(self) -> None: ...
|
|
236
202
|
|
|
237
203
|
@abstractmethod
|
|
238
|
-
def task_card(self) -> dict:
|
|
239
|
-
"""What an outside agent is handed about the session's task."""
|
|
204
|
+
def task_card(self) -> dict: ...
|
|
240
205
|
|
|
241
206
|
def state(self) -> dict:
|
|
242
207
|
return self.calls.into_state(self._rows())
|
|
243
208
|
|
|
244
|
-
def grader_state(self, report: Optional[str] = None) -> dict:
|
|
245
|
-
"""
|
|
246
|
-
dependency's as ``<alias>.<entity>``), the snapshot in use, the calls
|
|
247
|
-
the agent made (a read-only investigation writes nothing) and, under
|
|
248
|
-
``report``, the agent's own claim. Reserved keys: no world entity may
|
|
249
|
-
be named ``calls``, ``snapshot`` or ``report``."""
|
|
209
|
+
def grader_state(self, report: Optional[str] = None, journal: Optional[WorldJournal] = None) -> dict:
|
|
210
|
+
"""Keys beside the rows (``calls``, ``snapshot``, ``report``, ``gateway_*``) are reserved from entity names."""
|
|
250
211
|
state = self.calls.into_state(self._grader_rows())
|
|
212
|
+
if journal is not None:
|
|
213
|
+
state = journal.into_state(state, self._calls_view().rows(journal))
|
|
251
214
|
if report is not None:
|
|
252
215
|
state["report"] = report
|
|
253
216
|
return state
|
|
254
217
|
|
|
255
|
-
def
|
|
256
|
-
"""
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
218
|
+
def _calls_view(self) -> CallsView:
|
|
219
|
+
"""A tool call the door refused before it ran (a name not served or not allowed) is not a call."""
|
|
220
|
+
served = frozenset(self.served())
|
|
221
|
+
allowed = self.allowed
|
|
222
|
+
return CallsView(
|
|
223
|
+
self.cached.spec,
|
|
224
|
+
self.calls.redaction,
|
|
225
|
+
lambda tool: tool in served and (allowed is None or tool in allowed),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
def grade(
|
|
229
|
+
self,
|
|
230
|
+
report: Optional[str] = None,
|
|
231
|
+
context: Optional[dict] = None,
|
|
232
|
+
journal: Optional[WorldJournal] = None,
|
|
233
|
+
grader: Optional[dict] = None,
|
|
234
|
+
) -> dict:
|
|
235
|
+
"""A rubric stays pending: the host has no model, no key and no business calling one from a tenant's session.
|
|
236
|
+
Cross-world verifiers answer under ``verifiers``, apart from ``reward``, so an aggregate counts each once."""
|
|
237
|
+
if grader is not None:
|
|
238
|
+
return self.trial(grader, report, journal)
|
|
239
|
+
own = self._own_grade(report, journal)
|
|
271
240
|
verifiers = self._verifiers()
|
|
272
241
|
if not verifiers:
|
|
273
242
|
return own
|
|
@@ -275,23 +244,39 @@ class WorldRuntime(ABC):
|
|
|
275
244
|
graded = {grader.name: self._verifier_grade(grader, state, context or {}) for grader in verifiers}
|
|
276
245
|
return {**own, "verifiers": graded}
|
|
277
246
|
|
|
278
|
-
def _own_grade(self, report: Optional[str]) -> dict:
|
|
279
|
-
"""The session task's grade by its own grader, or null with the reason
|
|
280
|
-
when it has none; a bundled task's by the version's own graders."""
|
|
247
|
+
def _own_grade(self, report: Optional[str], journal: Optional[WorldJournal]) -> dict:
|
|
281
248
|
spec = self.task_spec
|
|
282
|
-
if spec is None
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
return bundled
|
|
249
|
+
bundled = self.bundled_grade(journal) if spec is None else None
|
|
250
|
+
if bundled is not None:
|
|
251
|
+
return bundled
|
|
286
252
|
grader = self._task_grader(spec)
|
|
287
253
|
if grader is None:
|
|
288
254
|
return Grade.ungraded().as_dict()
|
|
255
|
+
return self._graded(grader, report, journal)
|
|
256
|
+
|
|
257
|
+
def _graded(self, grader: TaskGrader, report: Optional[str], journal: Optional[WorldJournal]) -> dict:
|
|
289
258
|
try:
|
|
290
|
-
return grader.grade(self.grader_state(report)).as_dict()
|
|
259
|
+
return grader.grade(self.grader_state(report, journal if grader.reads_journal else None)).as_dict()
|
|
291
260
|
except GraderError as error:
|
|
292
261
|
fail("host", str(error))
|
|
293
262
|
raise AssertionError
|
|
294
263
|
|
|
264
|
+
def trial(self, table: dict, report: Optional[str], journal: Optional[WorldJournal]) -> dict:
|
|
265
|
+
parsed = _parse_grader(table, "/grader")
|
|
266
|
+
if parsed is None or parsed["kind"] == "rubric":
|
|
267
|
+
fail("host", "a candidate grader grades by assertions or python; a rubric is judged by the platform")
|
|
268
|
+
self._check_task_entities(parse_task_spec({"instruction": "try", "grader": table}))
|
|
269
|
+
sandbox, home = self._sandbox()
|
|
270
|
+
scratch = self.session_dir / "trial" / uuid.uuid4().hex
|
|
271
|
+
try:
|
|
272
|
+
verify = scratch / "verify.py"
|
|
273
|
+
scratch.mkdir(parents=True)
|
|
274
|
+
verify.write_text(parsed.get("source", ""), encoding="utf-8")
|
|
275
|
+
grader = TaskGrader(parsed, verify=verify, sandbox=sandbox, home=scratch / "home", explain=True)
|
|
276
|
+
return self._graded(grader, report, journal)
|
|
277
|
+
finally:
|
|
278
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
279
|
+
|
|
295
280
|
@staticmethod
|
|
296
281
|
def _verifier_grade(grader: TaskGrader, state: dict, context: dict) -> dict:
|
|
297
282
|
try:
|
|
@@ -303,24 +288,21 @@ class WorldRuntime(ABC):
|
|
|
303
288
|
return ConfinedVerifierSandbox(), self.session_dir / "verify"
|
|
304
289
|
|
|
305
290
|
def _task_grader(self, spec: Optional[TaskSpec]) -> Optional[TaskGrader]:
|
|
306
|
-
"""The session's grader: the spec's, with its verify.py where
|
|
307
|
-
:func:`write_task_dir` put it; else the bundled task's verify.py; else
|
|
308
|
-
none (an ungraded task grades null)."""
|
|
309
291
|
sandbox, home = self._sandbox()
|
|
310
292
|
if spec is not None:
|
|
311
|
-
|
|
312
|
-
return None
|
|
313
|
-
verify = self.session_dir / TASK_DIR / "checks" / "verify.py"
|
|
314
|
-
return TaskGrader(spec.grader, verify=verify, sandbox=sandbox, home=home)
|
|
293
|
+
return self._spec_grader(spec, sandbox, home)
|
|
315
294
|
verify = self._bundled_verifier()
|
|
316
295
|
if verify is None or not verify.is_file():
|
|
317
296
|
return None
|
|
318
297
|
return TaskGrader.for_verifier(verify, sandbox=sandbox, home=home)
|
|
319
298
|
|
|
299
|
+
def _spec_grader(self, spec: TaskSpec, sandbox: ConfinedVerifierSandbox, home: Path) -> Optional[TaskGrader]:
|
|
300
|
+
if spec.grader is None:
|
|
301
|
+
return None
|
|
302
|
+
verify = self.session_dir / TASK_DIR / "checks" / "verify.py"
|
|
303
|
+
return TaskGrader(spec.grader, verify=verify, sandbox=sandbox, home=home)
|
|
304
|
+
|
|
320
305
|
def _verifiers(self) -> list[TaskGrader]:
|
|
321
|
-
"""The task's cross-world verifiers anchored on this session, each a
|
|
322
|
-
scoped :class:`TaskGrader` with its verify.py where
|
|
323
|
-
:func:`write_task_dir` put it."""
|
|
324
306
|
spec = self.task_spec
|
|
325
307
|
if spec is None:
|
|
326
308
|
return []
|
|
@@ -338,19 +320,8 @@ class WorldRuntime(ABC):
|
|
|
338
320
|
for row in spec.verifiers
|
|
339
321
|
]
|
|
340
322
|
|
|
341
|
-
def credential(self) -> tuple[dict[str, str], dict[str, str]]:
|
|
342
|
-
"""(headers, query) carrying the world's own credential for this
|
|
343
|
-
session: the session token in the vendor's shape, or for an oauth2
|
|
344
|
-
world the grant the host minted for itself."""
|
|
345
|
-
assert self.app is not None, "a world of tools alone has no http surface to present a credential on"
|
|
346
|
-
return vendor_credential(self.app.router.spec, self.token or WORLD_AUTH_SENTINEL, grant=self.grant)
|
|
347
|
-
|
|
348
323
|
|
|
349
324
|
class TreeRuntime(WorldRuntime):
|
|
350
|
-
"""A schema world stored as itself: :class:`ToolsWorld` over a copy of the
|
|
351
|
-
version's template, beside a copy of each dependency's, graded by the
|
|
352
|
-
version's SQL verifiers unless the session's own task brings a grader."""
|
|
353
|
-
|
|
354
325
|
world: ToolsWorld
|
|
355
326
|
|
|
356
327
|
@classmethod
|
|
@@ -367,6 +338,14 @@ class TreeRuntime(WorldRuntime):
|
|
|
367
338
|
assert cached.tools is not None
|
|
368
339
|
return cached.tools.dependencies
|
|
369
340
|
|
|
341
|
+
@staticmethod
|
|
342
|
+
def live_state(session_dir: Path) -> Path:
|
|
343
|
+
return ToolsWorld.live_state(session_dir)
|
|
344
|
+
|
|
345
|
+
@staticmethod
|
|
346
|
+
def dependency_states(session_dir: Path) -> dict[str, Path]:
|
|
347
|
+
return ToolsWorld.dependency_states(session_dir)
|
|
348
|
+
|
|
370
349
|
def _start(self, dependencies: dict[str, DependencyWorld]) -> None:
|
|
371
350
|
tools = self.cached.tools
|
|
372
351
|
assert tools is not None
|
|
@@ -408,8 +387,8 @@ class TreeRuntime(WorldRuntime):
|
|
|
408
387
|
def _grader_rows(self) -> dict:
|
|
409
388
|
return self.world.grader_state()
|
|
410
389
|
|
|
411
|
-
def bundled_grade(self) -> dict | None:
|
|
412
|
-
return None if self.world.task.verify is not None else self.world.grade()
|
|
390
|
+
def bundled_grade(self, journal: WorldJournal | None = None) -> dict | None:
|
|
391
|
+
return None if self.world.task.verify is not None else self.world.grade(journal)
|
|
413
392
|
|
|
414
393
|
def _bundled_verifier(self) -> Path | None:
|
|
415
394
|
return self.world.task.verify
|
|
@@ -431,8 +410,7 @@ class TreeRuntime(WorldRuntime):
|
|
|
431
410
|
|
|
432
411
|
|
|
433
412
|
def scenario_dir(bundle: Path, tree: Path, task: str) -> Path | None:
|
|
434
|
-
"""
|
|
435
|
-
(where ``build_api_bundle`` writes them) then in the tree."""
|
|
413
|
+
"""The bundle root first, where ``build_api_bundle`` writes the scenarios, then the tree."""
|
|
436
414
|
for root in (bundle / SCENARIOS / task, tree / SCENARIOS / task):
|
|
437
415
|
if (root / "scenario.toml").is_file():
|
|
438
416
|
return root
|
|
@@ -440,17 +418,16 @@ def scenario_dir(bundle: Path, tree: Path, task: str) -> Path | None:
|
|
|
440
418
|
|
|
441
419
|
|
|
442
420
|
class HostWorldState(WorldState):
|
|
443
|
-
"""A session's world, opened by cloning the version's cached snapshot into a
|
|
444
|
-
file on the host's own volume — the reflink is the whole per-session cost.
|
|
445
|
-
Otherwise a plain :class:`WorldState`."""
|
|
446
|
-
|
|
447
421
|
def __init__(self, cached: CachedVersion, session_dir: Path):
|
|
448
422
|
self._session_dir = Path(session_dir)
|
|
449
423
|
self._session_dir.mkdir(parents=True, exist_ok=True)
|
|
450
424
|
self._cached = cached
|
|
451
|
-
self._database: Path | None = None
|
|
452
425
|
super().__init__(cached.tree, cached.served)
|
|
453
426
|
|
|
427
|
+
@staticmethod
|
|
428
|
+
def live_state(session_dir: Path) -> Path:
|
|
429
|
+
return Path(session_dir) / "state.db"
|
|
430
|
+
|
|
454
431
|
@property
|
|
455
432
|
def session_dir(self) -> Path:
|
|
456
433
|
return self._session_dir
|
|
@@ -461,26 +438,20 @@ class HostWorldState(WorldState):
|
|
|
461
438
|
self._lay(snapshot)
|
|
462
439
|
|
|
463
440
|
def _lay(self, source: Path) -> None:
|
|
464
|
-
"""A file of this session's own, cloned from the version's snapshot or a
|
|
465
|
-
verified export, and opened without the row audit."""
|
|
466
441
|
if self.store is not None:
|
|
467
442
|
self.store.close()
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
database = self._session_dir / f"{uuid.uuid4().hex}.db"
|
|
443
|
+
database = self.live_state(self._session_dir)
|
|
444
|
+
database.unlink(missing_ok=True)
|
|
471
445
|
self.store = WorldDatabase.open_copy(source, self.contract, database)
|
|
472
|
-
self._database = database
|
|
473
446
|
|
|
474
447
|
def export(self, destination: Path) -> Path:
|
|
475
|
-
"""
|
|
476
|
-
copied without holding the session, so its requests keep landing."""
|
|
448
|
+
"""Copied outside the lock, so the session's requests keep landing."""
|
|
477
449
|
with self._lock:
|
|
478
450
|
store = self.store
|
|
479
451
|
assert store is not None
|
|
480
452
|
return store.snapshot(destination)
|
|
481
453
|
|
|
482
454
|
def restore(self, export: SessionExport) -> None:
|
|
483
|
-
"""Continue from an export this host verified (:class:`SessionExport`)."""
|
|
484
455
|
path = SessionExport.required(export).path
|
|
485
456
|
with self._lock:
|
|
486
457
|
self._lay(path)
|
|
@@ -491,9 +462,8 @@ class HostWorldState(WorldState):
|
|
|
491
462
|
|
|
492
463
|
|
|
493
464
|
class SnapshotRuntime(WorldRuntime):
|
|
494
|
-
"""
|
|
495
|
-
snapshot
|
|
496
|
-
on top, graded by the scenario's ``verify.py``. READ PATH — see above."""
|
|
465
|
+
"""Read path for worlds still stored as api bundles; nothing new is stored in this shape. Delete it with
|
|
466
|
+
``CachedVersion.snapshot`` and the api-bundle arm of ``VersionCache._build``."""
|
|
497
467
|
|
|
498
468
|
state_: HostWorldState
|
|
499
469
|
|
|
@@ -507,6 +477,10 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
507
477
|
return None
|
|
508
478
|
return f"api handlers module {cached.settings.handlers.name}"
|
|
509
479
|
|
|
480
|
+
@staticmethod
|
|
481
|
+
def live_state(session_dir: Path) -> Path:
|
|
482
|
+
return HostWorldState.live_state(session_dir)
|
|
483
|
+
|
|
510
484
|
def _start(self, dependencies: dict[str, DependencyWorld]) -> None:
|
|
511
485
|
if dependencies:
|
|
512
486
|
fail(
|
|
@@ -517,10 +491,6 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
517
491
|
self.state_ = HostWorldState(self.cached, self.session_dir)
|
|
518
492
|
|
|
519
493
|
def _to_start(self) -> None:
|
|
520
|
-
"""The task's starting state on a fresh clone: the session task's seed,
|
|
521
|
-
else the bundled task's scenario overlay. The compiled snapshot the
|
|
522
|
-
session is cloned from IS the base state, so a scenario with no seed
|
|
523
|
-
overlay (the synthesized ``default``) needs nothing."""
|
|
524
494
|
spec = self.task_spec
|
|
525
495
|
if spec is not None:
|
|
526
496
|
if spec.seed is not None:
|
|
@@ -575,7 +545,7 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
575
545
|
def _grader_rows(self) -> dict:
|
|
576
546
|
return self._rows()
|
|
577
547
|
|
|
578
|
-
def bundled_grade(self) -> dict | None:
|
|
548
|
+
def bundled_grade(self, journal: WorldJournal | None = None) -> dict | None:
|
|
579
549
|
return None
|
|
580
550
|
|
|
581
551
|
def _bundled_verifier(self) -> Path | None:
|
|
@@ -608,19 +578,7 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
608
578
|
}
|
|
609
579
|
|
|
610
580
|
|
|
611
|
-
def client_grant(app: WorldApp | None, credential: str | None) -> str | None:
|
|
612
|
-
"""The bearer an ``oauth2`` world's routes take when the session itself
|
|
613
|
-
calls them (the page's api prefix); None for any other scheme, or when the
|
|
614
|
-
world refuses the client-credentials grant."""
|
|
615
|
-
if app is None or app.router.spec.scheme != "oauth2":
|
|
616
|
-
return None
|
|
617
|
-
issued = app.router.issue_token({"grant_type": "client_credentials", "client_secret": credential or ""}, credential)
|
|
618
|
-
return issued["access_token"] if issued else None
|
|
619
|
-
|
|
620
|
-
|
|
621
581
|
def tool_contracts(bundle: Path, prefix: str = "") -> dict[str, dict]:
|
|
622
|
-
"""Each compiled tool's ``{description, input}`` from the bundle's
|
|
623
|
-
``tool-schemas.json`` (refs inlined by the compiler), keyed by served name."""
|
|
624
582
|
path = bundle / "tool-schemas.json"
|
|
625
583
|
if not path.is_file():
|
|
626
584
|
return {}
|
|
@@ -632,11 +590,6 @@ def tool_contracts(bundle: Path, prefix: str = "") -> dict[str, dict]:
|
|
|
632
590
|
|
|
633
591
|
|
|
634
592
|
def task_card(world: ToolsWorld) -> dict:
|
|
635
|
-
"""What an outside agent is handed about the session's task: ``{task,
|
|
636
|
-
prompt, tools, contracts, verifiers, worlds}`` — ``tools`` the served names,
|
|
637
|
-
``contracts`` each one's description and input schema, ``verifiers`` the
|
|
638
|
-
root's that grade, ``worlds`` each dependency alias to the verifiers of its
|
|
639
|
-
own that grade (``slack/message_posted`` in the grade's ``rewards``)."""
|
|
640
593
|
task = world.task
|
|
641
594
|
contracts = tool_contracts(world.version.bundle)
|
|
642
595
|
for alias, dep in world.dependencies.items():
|