@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
"""The one engine a session runs on. :class:`WorldRuntime` is what the World
|
|
2
|
-
Host holds for a session opened in its process: the store, the task's
|
|
3
|
-
starting state, the HTTP surface when the tree declares one, and every
|
|
4
|
-
control verb (reset, seed, state, grade, call, export, restore, close). The
|
|
5
|
-
host asks the runtime; it never asks what shape the version was stored as.
|
|
6
|
-
|
|
7
|
-
Two runtimes exist because two stored shapes still do:
|
|
8
|
-
|
|
9
|
-
- :class:`TreeRuntime` — a schema world stored as itself (``gateway-env.toml``
|
|
10
|
-
with a ``[schema]`` table): a copy of the version's proven template,
|
|
11
|
-
executed through its handlers, beside a copy of each dependency world, its
|
|
12
|
-
routes (``connector.toml``) over the same store. This is the runtime every
|
|
13
|
-
new push opens on.
|
|
14
|
-
- :class:`SnapshotRuntime` — an api bundle (``gateway-api-world.toml`` over
|
|
15
|
-
``server/world``): a clone of the version's compiled snapshot behind its
|
|
16
|
-
routes. **READ PATH, marked for deletion**: it exists so the worlds the
|
|
17
|
-
platform already stores as bundles keep opening until the migration
|
|
18
|
-
(handoff step 5) has rewritten them as schema worlds; nothing new is
|
|
19
|
-
stored in this shape. Delete it, ``CachedVersion.snapshot`` and the
|
|
20
|
-
api-bundle arm of ``VersionCache._build`` together (step 6).
|
|
21
|
-
|
|
22
|
-
The shape is chosen once, in :meth:`WorldRuntime.open`, from what the version
|
|
23
|
-
cache loaded; from there the host sees one interface."""
|
|
24
|
-
|
|
25
1
|
from __future__ import annotations
|
|
26
2
|
|
|
27
3
|
import json
|
|
@@ -33,37 +9,37 @@ from pathlib import Path
|
|
|
33
9
|
from typing import TYPE_CHECKING, Any, Optional
|
|
34
10
|
|
|
35
11
|
from . import snapshot as snapshots
|
|
36
|
-
from .api import WorldApp, WorldState
|
|
37
|
-
from .call_log import CallLog
|
|
12
|
+
from .api import WorldApp, WorldState
|
|
13
|
+
from .call_log import CallJournal, CallLog
|
|
38
14
|
from .database import WorldDatabase
|
|
39
15
|
from .dependencies import Dependency
|
|
40
16
|
from .grading import Grade, GraderError, TaskGrader
|
|
17
|
+
from .journal import CallsView, WorldJournal
|
|
18
|
+
from .request_ledger import RequestOutbox
|
|
19
|
+
from .webhooks import WebhookPoll
|
|
41
20
|
from .served_names import ServedNames
|
|
42
|
-
from .session_files import SessionExport
|
|
21
|
+
from .session_files import CALLS_JOURNAL, LEDGER_JOURNAL, SessionExport
|
|
43
22
|
from .snapshot import SCENARIOS
|
|
44
|
-
from .tasks import
|
|
23
|
+
from .tasks import (
|
|
24
|
+
TASK_DIR,
|
|
25
|
+
TaskSpec,
|
|
26
|
+
_parse_grader,
|
|
27
|
+
parse_task_spec,
|
|
28
|
+
unknown_task_entities,
|
|
29
|
+
verifier_source,
|
|
30
|
+
write_task_dir,
|
|
31
|
+
)
|
|
45
32
|
from .tools_world import DependencyWorld, ToolsWorld, ToolsWorldState, inline_task, served_with_dependencies
|
|
46
33
|
from .validation import fail
|
|
47
34
|
from .workers import ConfinedVerifierSandbox
|
|
48
35
|
|
|
49
36
|
if TYPE_CHECKING:
|
|
50
|
-
from .
|
|
51
|
-
|
|
52
|
-
# The router still authorizes by it, so the host's own calls carry it.
|
|
53
|
-
WORLD_AUTH_SENTINEL = "gateway-session"
|
|
37
|
+
from .host_versions import CachedVersion
|
|
54
38
|
|
|
55
39
|
Restorer = Callable[[Callable[[SessionExport], None]], bool]
|
|
56
40
|
|
|
57
41
|
|
|
58
42
|
class WorldRuntime(ABC):
|
|
59
|
-
"""One session's world, in process. Built by :meth:`open`; the host keeps
|
|
60
|
-
it on the session and calls nothing else of the world's. ``app`` is the
|
|
61
|
-
routes of connector.toml over the session's store, None for a world of
|
|
62
|
-
tools alone; ``calls`` is the one call log its routes and its tools write;
|
|
63
|
-
``grant`` is an ``oauth2`` world's client-credentials
|
|
64
|
-
grant the host minted for itself, so a caller the session token admits is
|
|
65
|
-
presented to the world as a granted client."""
|
|
66
|
-
|
|
67
43
|
def __init__(
|
|
68
44
|
self,
|
|
69
45
|
cached: CachedVersion,
|
|
@@ -72,6 +48,7 @@ class WorldRuntime(ABC):
|
|
|
72
48
|
task: str | None,
|
|
73
49
|
spec: TaskSpec | None,
|
|
74
50
|
token: str | None,
|
|
51
|
+
journaled: bool = False,
|
|
75
52
|
) -> None:
|
|
76
53
|
self.cached = cached
|
|
77
54
|
self.session_dir = Path(session_dir)
|
|
@@ -79,24 +56,31 @@ class WorldRuntime(ABC):
|
|
|
79
56
|
self.task_spec = spec
|
|
80
57
|
self.token = token
|
|
81
58
|
self.app: WorldApp | None = None
|
|
82
|
-
|
|
83
|
-
self.
|
|
59
|
+
journal = CallJournal(self.session_dir / CALLS_JOURNAL) if journaled else None
|
|
60
|
+
outbox = RequestOutbox(CallJournal(self.session_dir / LEDGER_JOURNAL) if journaled else None)
|
|
61
|
+
spec = cached.spec
|
|
62
|
+
self.calls = CallLog.for_spec(spec, journal, outbox, token)
|
|
84
63
|
self.restored = False
|
|
85
64
|
|
|
86
65
|
@classmethod
|
|
87
66
|
@abstractmethod
|
|
88
|
-
def served_by(cls, cached: CachedVersion, dependencies: dict[str, DependencyWorld]) -> list[str]:
|
|
89
|
-
"""Every tool a session task on ``cached`` may allow."""
|
|
67
|
+
def served_by(cls, cached: CachedVersion, dependencies: dict[str, DependencyWorld]) -> list[str]: ...
|
|
90
68
|
|
|
91
69
|
@classmethod
|
|
92
70
|
@abstractmethod
|
|
93
71
|
def authored_code(cls, cached: CachedVersion) -> str | None:
|
|
94
|
-
"""
|
|
95
|
-
it), None when it runs none."""
|
|
72
|
+
"""World-authored code ``cached`` runs, so only a version worker may open it; None when it runs none."""
|
|
96
73
|
|
|
97
74
|
@classmethod
|
|
98
75
|
def declared_dependencies(cls, cached: CachedVersion) -> dict[str, Dependency]:
|
|
99
|
-
|
|
76
|
+
return {}
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def live_state(session_dir: Path) -> Path: ...
|
|
81
|
+
|
|
82
|
+
@staticmethod
|
|
83
|
+
def dependency_states(session_dir: Path) -> dict[str, Path]:
|
|
100
84
|
return {}
|
|
101
85
|
|
|
102
86
|
@classmethod
|
|
@@ -110,11 +94,10 @@ class WorldRuntime(ABC):
|
|
|
110
94
|
token: str | None,
|
|
111
95
|
dependencies: dict[str, DependencyWorld],
|
|
112
96
|
restore: Restorer,
|
|
97
|
+
journaled: bool = False,
|
|
113
98
|
) -> WorldRuntime:
|
|
114
|
-
"""
|
|
115
|
-
|
|
116
|
-
attached. A start that fails leaves no session directory behind."""
|
|
117
|
-
runtime = cached.runtime_class(cached, session_dir, task=task, spec=spec, token=token)
|
|
99
|
+
"""``journaled`` keeps the calls on disk, so a version worker that dies leaves them to the next one."""
|
|
100
|
+
runtime = cached.runtime_class(cached, session_dir, task=task, spec=spec, token=token, journaled=journaled)
|
|
118
101
|
runtime._start(dependencies)
|
|
119
102
|
try:
|
|
120
103
|
if spec is not None:
|
|
@@ -141,7 +124,6 @@ class WorldRuntime(ABC):
|
|
|
141
124
|
routes_closed=self.allowed is not None,
|
|
142
125
|
calls=self.calls,
|
|
143
126
|
)
|
|
144
|
-
self.grant = client_grant(self.app, self.token)
|
|
145
127
|
|
|
146
128
|
@property
|
|
147
129
|
def allowed(self) -> frozenset[str] | None:
|
|
@@ -149,10 +131,7 @@ class WorldRuntime(ABC):
|
|
|
149
131
|
return frozenset(spec.tools) if spec is not None and spec.tools is not None else None
|
|
150
132
|
|
|
151
133
|
def _check_task_entities(self, spec: TaskSpec) -> None:
|
|
152
|
-
"""A
|
|
153
|
-
names. The seed would fail on its own; an assertion over an unknown
|
|
154
|
-
entity would not, selecting no rows, so ``absent`` or ``count: eq 0``
|
|
155
|
-
checks would pass on a typo and grade a task that can never be graded."""
|
|
134
|
+
"""A check over an unknown entity selects no rows: ``absent`` or ``count: eq 0`` would pass on a typo."""
|
|
156
135
|
defined = self.entity_names()
|
|
157
136
|
unknown = unknown_task_entities(spec, defined)
|
|
158
137
|
if unknown:
|
|
@@ -165,22 +144,19 @@ class WorldRuntime(ABC):
|
|
|
165
144
|
|
|
166
145
|
@abstractmethod
|
|
167
146
|
def _start(self, dependencies: dict[str, DependencyWorld]) -> None:
|
|
168
|
-
"""
|
|
147
|
+
"""A failure here leaves nothing to close: :meth:`open` runs it outside its cleanup."""
|
|
169
148
|
|
|
170
149
|
def _to_start(self) -> None:
|
|
171
150
|
"""Lay the task's starting state on the store, when :meth:`_start` did not."""
|
|
172
151
|
|
|
173
152
|
@abstractmethod
|
|
174
|
-
def _app_state(self) -> Any:
|
|
175
|
-
"""The :class:`api.WorldState` the routes read and write."""
|
|
153
|
+
def _app_state(self) -> Any: ...
|
|
176
154
|
|
|
177
155
|
@abstractmethod
|
|
178
|
-
def entity_names(self) -> list[str]:
|
|
179
|
-
"""Every entity a grader may name (a dependency's as ``alias.entity``)."""
|
|
156
|
+
def entity_names(self) -> list[str]: ...
|
|
180
157
|
|
|
181
158
|
@abstractmethod
|
|
182
|
-
def served(self) -> list[str]:
|
|
183
|
-
"""Every name :meth:`call` answers: the declared tools, a dependency's as ``alias.tool``."""
|
|
159
|
+
def served(self) -> list[str]: ...
|
|
184
160
|
|
|
185
161
|
@abstractmethod
|
|
186
162
|
def reset(self) -> dict: ...
|
|
@@ -189,85 +165,85 @@ class WorldRuntime(ABC):
|
|
|
189
165
|
def seed(self, rows: dict, mode: str, redact: str) -> dict: ...
|
|
190
166
|
|
|
191
167
|
@abstractmethod
|
|
192
|
-
def _rows(self) -> dict:
|
|
193
|
-
"""The state without the calls: rows by entity, and what the store started from."""
|
|
168
|
+
def _rows(self) -> dict: ...
|
|
194
169
|
|
|
195
170
|
@abstractmethod
|
|
196
|
-
def _grader_rows(self) -> dict:
|
|
197
|
-
"""The rows as a session task's grader reads them (keyed by primary key)."""
|
|
171
|
+
def _grader_rows(self) -> dict: ...
|
|
198
172
|
|
|
199
173
|
@abstractmethod
|
|
200
|
-
def bundled_grade(self) -> dict | None:
|
|
201
|
-
"""
|
|
202
|
-
(a schema world's SQL verifiers), or None when the version grades a
|
|
203
|
-
bundled task through a ``verify.py`` the :class:`TaskGrader` runs."""
|
|
174
|
+
def bundled_grade(self, journal: WorldJournal | None = None) -> dict | None:
|
|
175
|
+
"""None means the bundled task grades by its ``verify.py`` (:class:`TaskGrader`), not that it is ungraded."""
|
|
204
176
|
|
|
205
177
|
@abstractmethod
|
|
206
|
-
def _bundled_verifier(self) -> Path | None:
|
|
207
|
-
"""The ``verify.py`` of the bundled task, when the version carries one."""
|
|
178
|
+
def _bundled_verifier(self) -> Path | None: ...
|
|
208
179
|
|
|
209
180
|
@abstractmethod
|
|
210
181
|
def call(self, tool: str, args: dict) -> dict:
|
|
211
|
-
"""A
|
|
212
|
-
refused with the api URL to call it over: routes are never tools."""
|
|
182
|
+
"""A route's name is refused with the api URL to call it over: routes are never tools."""
|
|
213
183
|
|
|
214
184
|
@abstractmethod
|
|
215
|
-
def _export_state(self, destination: Path) -> Path:
|
|
216
|
-
"""The store's copy of the session's state at ``destination``."""
|
|
185
|
+
def _export_state(self, destination: Path) -> Path: ...
|
|
217
186
|
|
|
218
187
|
@abstractmethod
|
|
219
188
|
def _restore_state(self, export: SessionExport) -> None:
|
|
220
189
|
"""The session's store laid from a verified export, without a row audit."""
|
|
221
190
|
|
|
222
191
|
def export(self, destination: Path) -> Path:
|
|
223
|
-
"""The session's state as one database file, with its call log in it, so a
|
|
224
|
-
reopen resumes both (:class:`SessionExport`)."""
|
|
225
192
|
path = self._export_state(destination)
|
|
226
193
|
SessionExport.carry_calls(path, self.calls.describe()["calls"], self.cached.engine)
|
|
227
194
|
return path
|
|
228
195
|
|
|
229
196
|
def restore(self, export: SessionExport) -> None:
|
|
230
|
-
"""Continue from an export this host verified, its call log with it."""
|
|
231
197
|
self._restore_state(export)
|
|
232
198
|
self.calls.resume(export.calls)
|
|
199
|
+
self.calls.outbox.resume(export.unsent)
|
|
233
200
|
|
|
234
201
|
@abstractmethod
|
|
235
202
|
def close(self) -> None: ...
|
|
236
203
|
|
|
237
204
|
@abstractmethod
|
|
238
|
-
def task_card(self) -> dict:
|
|
239
|
-
"""What an outside agent is handed about the session's task."""
|
|
205
|
+
def task_card(self) -> dict: ...
|
|
240
206
|
|
|
241
207
|
def state(self) -> dict:
|
|
242
208
|
return self.calls.into_state(self._rows())
|
|
243
209
|
|
|
244
|
-
def
|
|
245
|
-
"""The
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
210
|
+
def webhooks(self, poll: WebhookPoll) -> dict:
|
|
211
|
+
"""The subscriptions the world's ``[webhooks]`` finds (none declared: ``declared`` false), a page
|
|
212
|
+
of deliveries, and how many of the poll's answers the ledger kept."""
|
|
213
|
+
outbox = self.app.state.webhooks if self.app is not None else None
|
|
214
|
+
return self.calls.webhook_page(outbox, poll)
|
|
215
|
+
|
|
216
|
+
def grader_state(self, report: Optional[str] = None, journal: Optional[WorldJournal] = None) -> dict:
|
|
217
|
+
"""Keys beside the rows (``calls``, ``snapshot``, ``report``, ``gateway_*``) are reserved from entity names."""
|
|
250
218
|
state = self.calls.into_state(self._grader_rows())
|
|
219
|
+
if journal is not None:
|
|
220
|
+
state = journal.into_state(state, self._calls_view().rows(journal))
|
|
251
221
|
if report is not None:
|
|
252
222
|
state["report"] = report
|
|
253
223
|
return state
|
|
254
224
|
|
|
255
|
-
def
|
|
256
|
-
"""
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
225
|
+
def _calls_view(self) -> CallsView:
|
|
226
|
+
"""A tool call the door refused before it ran (a name not served or not allowed) is not a call."""
|
|
227
|
+
served = frozenset(self.served())
|
|
228
|
+
allowed = self.allowed
|
|
229
|
+
return CallsView(
|
|
230
|
+
self.cached.spec,
|
|
231
|
+
self.calls.redaction,
|
|
232
|
+
lambda tool: tool in served and (allowed is None or tool in allowed),
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
def grade(
|
|
236
|
+
self,
|
|
237
|
+
report: Optional[str] = None,
|
|
238
|
+
context: Optional[dict] = None,
|
|
239
|
+
journal: Optional[WorldJournal] = None,
|
|
240
|
+
grader: Optional[dict] = None,
|
|
241
|
+
) -> dict:
|
|
242
|
+
"""A rubric stays pending: the host has no model, no key and no business calling one from a tenant's session.
|
|
243
|
+
Cross-world verifiers answer under ``verifiers``, apart from ``reward``, so an aggregate counts each once."""
|
|
244
|
+
if grader is not None:
|
|
245
|
+
return self.trial(grader, report, journal)
|
|
246
|
+
own = self._own_grade(report, journal)
|
|
271
247
|
verifiers = self._verifiers()
|
|
272
248
|
if not verifiers:
|
|
273
249
|
return own
|
|
@@ -275,23 +251,39 @@ class WorldRuntime(ABC):
|
|
|
275
251
|
graded = {grader.name: self._verifier_grade(grader, state, context or {}) for grader in verifiers}
|
|
276
252
|
return {**own, "verifiers": graded}
|
|
277
253
|
|
|
278
|
-
def _own_grade(self, report: Optional[str]) -> dict:
|
|
279
|
-
"""The session task's grade by its own grader, or null with the reason
|
|
280
|
-
when it has none; a bundled task's by the version's own graders."""
|
|
254
|
+
def _own_grade(self, report: Optional[str], journal: Optional[WorldJournal]) -> dict:
|
|
281
255
|
spec = self.task_spec
|
|
282
|
-
if spec is None
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
return bundled
|
|
256
|
+
bundled = self.bundled_grade(journal) if spec is None else None
|
|
257
|
+
if bundled is not None:
|
|
258
|
+
return bundled
|
|
286
259
|
grader = self._task_grader(spec)
|
|
287
260
|
if grader is None:
|
|
288
261
|
return Grade.ungraded().as_dict()
|
|
262
|
+
return self._graded(grader, report, journal)
|
|
263
|
+
|
|
264
|
+
def _graded(self, grader: TaskGrader, report: Optional[str], journal: Optional[WorldJournal]) -> dict:
|
|
289
265
|
try:
|
|
290
|
-
return grader.grade(self.grader_state(report)).as_dict()
|
|
266
|
+
return grader.grade(self.grader_state(report, journal if grader.reads_journal else None)).as_dict()
|
|
291
267
|
except GraderError as error:
|
|
292
268
|
fail("host", str(error))
|
|
293
269
|
raise AssertionError
|
|
294
270
|
|
|
271
|
+
def trial(self, table: dict, report: Optional[str], journal: Optional[WorldJournal]) -> dict:
|
|
272
|
+
parsed = _parse_grader(table, "/grader")
|
|
273
|
+
if parsed is None or parsed["kind"] == "rubric":
|
|
274
|
+
fail("host", "a candidate grader grades by assertions or python; a rubric is judged by the platform")
|
|
275
|
+
self._check_task_entities(parse_task_spec({"instruction": "try", "grader": table}))
|
|
276
|
+
sandbox, home = self._sandbox()
|
|
277
|
+
scratch = self.session_dir / "trial" / uuid.uuid4().hex
|
|
278
|
+
try:
|
|
279
|
+
verify = scratch / "verify.py"
|
|
280
|
+
scratch.mkdir(parents=True)
|
|
281
|
+
verify.write_text(parsed.get("source", ""), encoding="utf-8")
|
|
282
|
+
grader = TaskGrader(parsed, verify=verify, sandbox=sandbox, home=scratch / "home", explain=True)
|
|
283
|
+
return self._graded(grader, report, journal)
|
|
284
|
+
finally:
|
|
285
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
286
|
+
|
|
295
287
|
@staticmethod
|
|
296
288
|
def _verifier_grade(grader: TaskGrader, state: dict, context: dict) -> dict:
|
|
297
289
|
try:
|
|
@@ -303,24 +295,21 @@ class WorldRuntime(ABC):
|
|
|
303
295
|
return ConfinedVerifierSandbox(), self.session_dir / "verify"
|
|
304
296
|
|
|
305
297
|
def _task_grader(self, spec: Optional[TaskSpec]) -> Optional[TaskGrader]:
|
|
306
|
-
"""The session's grader: the spec's, with its verify.py where
|
|
307
|
-
:func:`write_task_dir` put it; else the bundled task's verify.py; else
|
|
308
|
-
none (an ungraded task grades null)."""
|
|
309
298
|
sandbox, home = self._sandbox()
|
|
310
299
|
if spec is not None:
|
|
311
|
-
|
|
312
|
-
return None
|
|
313
|
-
verify = self.session_dir / TASK_DIR / "checks" / "verify.py"
|
|
314
|
-
return TaskGrader(spec.grader, verify=verify, sandbox=sandbox, home=home)
|
|
300
|
+
return self._spec_grader(spec, sandbox, home)
|
|
315
301
|
verify = self._bundled_verifier()
|
|
316
302
|
if verify is None or not verify.is_file():
|
|
317
303
|
return None
|
|
318
304
|
return TaskGrader.for_verifier(verify, sandbox=sandbox, home=home)
|
|
319
305
|
|
|
306
|
+
def _spec_grader(self, spec: TaskSpec, sandbox: ConfinedVerifierSandbox, home: Path) -> Optional[TaskGrader]:
|
|
307
|
+
if spec.grader is None:
|
|
308
|
+
return None
|
|
309
|
+
verify = self.session_dir / TASK_DIR / "checks" / "verify.py"
|
|
310
|
+
return TaskGrader(spec.grader, verify=verify, sandbox=sandbox, home=home)
|
|
311
|
+
|
|
320
312
|
def _verifiers(self) -> list[TaskGrader]:
|
|
321
|
-
"""The task's cross-world verifiers anchored on this session, each a
|
|
322
|
-
scoped :class:`TaskGrader` with its verify.py where
|
|
323
|
-
:func:`write_task_dir` put it."""
|
|
324
313
|
spec = self.task_spec
|
|
325
314
|
if spec is None:
|
|
326
315
|
return []
|
|
@@ -338,19 +327,8 @@ class WorldRuntime(ABC):
|
|
|
338
327
|
for row in spec.verifiers
|
|
339
328
|
]
|
|
340
329
|
|
|
341
|
-
def credential(self) -> tuple[dict[str, str], dict[str, str]]:
|
|
342
|
-
"""(headers, query) carrying the world's own credential for this
|
|
343
|
-
session: the session token in the vendor's shape, or for an oauth2
|
|
344
|
-
world the grant the host minted for itself."""
|
|
345
|
-
assert self.app is not None, "a world of tools alone has no http surface to present a credential on"
|
|
346
|
-
return vendor_credential(self.app.router.spec, self.token or WORLD_AUTH_SENTINEL, grant=self.grant)
|
|
347
|
-
|
|
348
330
|
|
|
349
331
|
class TreeRuntime(WorldRuntime):
|
|
350
|
-
"""A schema world stored as itself: :class:`ToolsWorld` over a copy of the
|
|
351
|
-
version's template, beside a copy of each dependency's, graded by the
|
|
352
|
-
version's SQL verifiers unless the session's own task brings a grader."""
|
|
353
|
-
|
|
354
332
|
world: ToolsWorld
|
|
355
333
|
|
|
356
334
|
@classmethod
|
|
@@ -367,6 +345,14 @@ class TreeRuntime(WorldRuntime):
|
|
|
367
345
|
assert cached.tools is not None
|
|
368
346
|
return cached.tools.dependencies
|
|
369
347
|
|
|
348
|
+
@staticmethod
|
|
349
|
+
def live_state(session_dir: Path) -> Path:
|
|
350
|
+
return ToolsWorld.live_state(session_dir)
|
|
351
|
+
|
|
352
|
+
@staticmethod
|
|
353
|
+
def dependency_states(session_dir: Path) -> dict[str, Path]:
|
|
354
|
+
return ToolsWorld.dependency_states(session_dir)
|
|
355
|
+
|
|
370
356
|
def _start(self, dependencies: dict[str, DependencyWorld]) -> None:
|
|
371
357
|
tools = self.cached.tools
|
|
372
358
|
assert tools is not None
|
|
@@ -408,8 +394,8 @@ class TreeRuntime(WorldRuntime):
|
|
|
408
394
|
def _grader_rows(self) -> dict:
|
|
409
395
|
return self.world.grader_state()
|
|
410
396
|
|
|
411
|
-
def bundled_grade(self) -> dict | None:
|
|
412
|
-
return None if self.world.task.verify is not None else self.world.grade()
|
|
397
|
+
def bundled_grade(self, journal: WorldJournal | None = None) -> dict | None:
|
|
398
|
+
return None if self.world.task.verify is not None else self.world.grade(journal)
|
|
413
399
|
|
|
414
400
|
def _bundled_verifier(self) -> Path | None:
|
|
415
401
|
return self.world.task.verify
|
|
@@ -431,8 +417,7 @@ class TreeRuntime(WorldRuntime):
|
|
|
431
417
|
|
|
432
418
|
|
|
433
419
|
def scenario_dir(bundle: Path, tree: Path, task: str) -> Path | None:
|
|
434
|
-
"""
|
|
435
|
-
(where ``build_api_bundle`` writes them) then in the tree."""
|
|
420
|
+
"""The bundle root first, where ``build_api_bundle`` writes the scenarios, then the tree."""
|
|
436
421
|
for root in (bundle / SCENARIOS / task, tree / SCENARIOS / task):
|
|
437
422
|
if (root / "scenario.toml").is_file():
|
|
438
423
|
return root
|
|
@@ -440,17 +425,16 @@ def scenario_dir(bundle: Path, tree: Path, task: str) -> Path | None:
|
|
|
440
425
|
|
|
441
426
|
|
|
442
427
|
class HostWorldState(WorldState):
|
|
443
|
-
"""A session's world, opened by cloning the version's cached snapshot into a
|
|
444
|
-
file on the host's own volume — the reflink is the whole per-session cost.
|
|
445
|
-
Otherwise a plain :class:`WorldState`."""
|
|
446
|
-
|
|
447
428
|
def __init__(self, cached: CachedVersion, session_dir: Path):
|
|
448
429
|
self._session_dir = Path(session_dir)
|
|
449
430
|
self._session_dir.mkdir(parents=True, exist_ok=True)
|
|
450
431
|
self._cached = cached
|
|
451
|
-
self._database: Path | None = None
|
|
452
432
|
super().__init__(cached.tree, cached.served)
|
|
453
433
|
|
|
434
|
+
@staticmethod
|
|
435
|
+
def live_state(session_dir: Path) -> Path:
|
|
436
|
+
return Path(session_dir) / "state.db"
|
|
437
|
+
|
|
454
438
|
@property
|
|
455
439
|
def session_dir(self) -> Path:
|
|
456
440
|
return self._session_dir
|
|
@@ -461,26 +445,20 @@ class HostWorldState(WorldState):
|
|
|
461
445
|
self._lay(snapshot)
|
|
462
446
|
|
|
463
447
|
def _lay(self, source: Path) -> None:
|
|
464
|
-
"""A file of this session's own, cloned from the version's snapshot or a
|
|
465
|
-
verified export, and opened without the row audit."""
|
|
466
448
|
if self.store is not None:
|
|
467
449
|
self.store.close()
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
database = self._session_dir / f"{uuid.uuid4().hex}.db"
|
|
450
|
+
database = self.live_state(self._session_dir)
|
|
451
|
+
database.unlink(missing_ok=True)
|
|
471
452
|
self.store = WorldDatabase.open_copy(source, self.contract, database)
|
|
472
|
-
self._database = database
|
|
473
453
|
|
|
474
454
|
def export(self, destination: Path) -> Path:
|
|
475
|
-
"""
|
|
476
|
-
copied without holding the session, so its requests keep landing."""
|
|
455
|
+
"""Copied outside the lock, so the session's requests keep landing."""
|
|
477
456
|
with self._lock:
|
|
478
457
|
store = self.store
|
|
479
458
|
assert store is not None
|
|
480
459
|
return store.snapshot(destination)
|
|
481
460
|
|
|
482
461
|
def restore(self, export: SessionExport) -> None:
|
|
483
|
-
"""Continue from an export this host verified (:class:`SessionExport`)."""
|
|
484
462
|
path = SessionExport.required(export).path
|
|
485
463
|
with self._lock:
|
|
486
464
|
self._lay(path)
|
|
@@ -491,9 +469,8 @@ class HostWorldState(WorldState):
|
|
|
491
469
|
|
|
492
470
|
|
|
493
471
|
class SnapshotRuntime(WorldRuntime):
|
|
494
|
-
"""
|
|
495
|
-
snapshot
|
|
496
|
-
on top, graded by the scenario's ``verify.py``. READ PATH — see above."""
|
|
472
|
+
"""Read path for worlds still stored as api bundles; nothing new is stored in this shape. Delete it with
|
|
473
|
+
``CachedVersion.snapshot`` and the api-bundle arm of ``VersionCache._build``."""
|
|
497
474
|
|
|
498
475
|
state_: HostWorldState
|
|
499
476
|
|
|
@@ -507,6 +484,10 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
507
484
|
return None
|
|
508
485
|
return f"api handlers module {cached.settings.handlers.name}"
|
|
509
486
|
|
|
487
|
+
@staticmethod
|
|
488
|
+
def live_state(session_dir: Path) -> Path:
|
|
489
|
+
return HostWorldState.live_state(session_dir)
|
|
490
|
+
|
|
510
491
|
def _start(self, dependencies: dict[str, DependencyWorld]) -> None:
|
|
511
492
|
if dependencies:
|
|
512
493
|
fail(
|
|
@@ -517,10 +498,6 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
517
498
|
self.state_ = HostWorldState(self.cached, self.session_dir)
|
|
518
499
|
|
|
519
500
|
def _to_start(self) -> None:
|
|
520
|
-
"""The task's starting state on a fresh clone: the session task's seed,
|
|
521
|
-
else the bundled task's scenario overlay. The compiled snapshot the
|
|
522
|
-
session is cloned from IS the base state, so a scenario with no seed
|
|
523
|
-
overlay (the synthesized ``default``) needs nothing."""
|
|
524
501
|
spec = self.task_spec
|
|
525
502
|
if spec is not None:
|
|
526
503
|
if spec.seed is not None:
|
|
@@ -575,7 +552,7 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
575
552
|
def _grader_rows(self) -> dict:
|
|
576
553
|
return self._rows()
|
|
577
554
|
|
|
578
|
-
def bundled_grade(self) -> dict | None:
|
|
555
|
+
def bundled_grade(self, journal: WorldJournal | None = None) -> dict | None:
|
|
579
556
|
return None
|
|
580
557
|
|
|
581
558
|
def _bundled_verifier(self) -> Path | None:
|
|
@@ -608,19 +585,7 @@ class SnapshotRuntime(WorldRuntime):
|
|
|
608
585
|
}
|
|
609
586
|
|
|
610
587
|
|
|
611
|
-
def client_grant(app: WorldApp | None, credential: str | None) -> str | None:
|
|
612
|
-
"""The bearer an ``oauth2`` world's routes take when the session itself
|
|
613
|
-
calls them (the page's api prefix); None for any other scheme, or when the
|
|
614
|
-
world refuses the client-credentials grant."""
|
|
615
|
-
if app is None or app.router.spec.scheme != "oauth2":
|
|
616
|
-
return None
|
|
617
|
-
issued = app.router.issue_token({"grant_type": "client_credentials", "client_secret": credential or ""}, credential)
|
|
618
|
-
return issued["access_token"] if issued else None
|
|
619
|
-
|
|
620
|
-
|
|
621
588
|
def tool_contracts(bundle: Path, prefix: str = "") -> dict[str, dict]:
|
|
622
|
-
"""Each compiled tool's ``{description, input}`` from the bundle's
|
|
623
|
-
``tool-schemas.json`` (refs inlined by the compiler), keyed by served name."""
|
|
624
589
|
path = bundle / "tool-schemas.json"
|
|
625
590
|
if not path.is_file():
|
|
626
591
|
return {}
|
|
@@ -632,11 +597,6 @@ def tool_contracts(bundle: Path, prefix: str = "") -> dict[str, dict]:
|
|
|
632
597
|
|
|
633
598
|
|
|
634
599
|
def task_card(world: ToolsWorld) -> dict:
|
|
635
|
-
"""What an outside agent is handed about the session's task: ``{task,
|
|
636
|
-
prompt, tools, contracts, verifiers, worlds}`` — ``tools`` the served names,
|
|
637
|
-
``contracts`` each one's description and input schema, ``verifiers`` the
|
|
638
|
-
root's that grade, ``worlds`` each dependency alias to the verifiers of its
|
|
639
|
-
own that grade (``slack/message_posted`` in the grade's ``rewards``)."""
|
|
640
600
|
task = world.task
|
|
641
601
|
contracts = tool_contracts(world.version.bundle)
|
|
642
602
|
for alias, dep in world.dependencies.items():
|
|
@@ -61,7 +61,8 @@ hosted call into such a handler answers 400 {error: "handler", detail, handler}:
|
|
|
61
61
|
{store}
|
|
62
62
|
|
|
63
63
|
A handler that raises AttributeError, TypeError or NameError is answered 400 with
|
|
64
|
-
that structured body;
|
|
64
|
+
that structured body; `raise Refusal(404, body)` (from gateway_env) answers that 4xx
|
|
65
|
+
and body as the vendor's refusal; any other exception is answered 500 {error: "engine", ...}.
|
|
65
66
|
`schema init --connector custom --with-example-tool` writes the example in full.
|
|
66
67
|
"""
|
|
67
68
|
'''
|