@withgateway/sdk 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +13267 -5869
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +13272 -5874
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +83 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -28
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +4355 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/index.mjs +4243 -0
- package/dist/hooks/index.mjs.map +1 -0
- package/dist/{index-Dpo-t8iO.d.ts → index-1xnxt0Cc.d.ts} +58 -2
- package/dist/index-B0R3IgsV.d.mts +1279 -0
- package/dist/index-B0R3IgsV.d.ts +1279 -0
- package/dist/{index-B5HUQQDE.d.mts → index-C1wDEwH4.d.mts} +58 -2
- package/dist/index.d.mts +4 -3
- package/dist/index.d.ts +4 -3
- package/dist/index.js +6698 -1606
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6793 -1713
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +72 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +68 -17
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +47 -3
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +42 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +390 -27
- package/dist/worlds/index.d.ts +390 -27
- package/dist/worlds/index.js +2291 -366
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +2281 -356
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +7 -1
- package/runtime/gatewaysdk/environment/_bundle.py +14 -24
- package/runtime/gatewaysdk/environment/_materialize.py +3 -3
- package/runtime/gatewaysdk/environment/schema/api.py +153 -20
- package/runtime/gatewaysdk/environment/schema/compiler.py +14 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +171 -535
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +156 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +6 -5
- package/runtime/gatewaysdk/environment/schema/tasks.py +12 -2
- package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +44 -13
- package/runtime/gatewaysdk/environment/world.py +391 -0
- package/skills/hooks-getting-started/SKILL.md +156 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: hooks-getting-started
|
|
3
|
+
description: Rules around an agent's tool calls from one gateway-hooks.toml — the alias door (personas in, real values out), identities that arrive from the web (mint_inbound), redaction of anything (kinds, literal values, patterns), per-task overrides, coding-agent and agent-SDK wiring, and how to check a file before an agent runs it. Use when an agent must not see a real person's identifiers, when web results must be scrubbed, or when wiring gateway hooks into Claude Code or an agent SDK.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Hooks: getting started
|
|
7
|
+
|
|
8
|
+
One file, `gateway-hooks.toml`, holds the rules that run around an agent's tool calls. The
|
|
9
|
+
same file is attached in code by an agent SDK (`HookRun.from_config` / `HookRun.fromConfig`),
|
|
10
|
+
run as a coding agent's hook (`gateway hooks run`), and validated without an agent
|
|
11
|
+
(`gateway hooks check`). Nothing in it is automatic: every behaviour is a rule you wrote or a
|
|
12
|
+
knob with a default, and a task can override the file.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
gateway hooks guide # this page
|
|
16
|
+
gateway hooks check gateway-hooks.toml # validate, build every handler, dry-run one event
|
|
17
|
+
gateway hooks run # the command a coding agent names in its settings
|
|
18
|
+
gateway hooks legend <run dir> # persona -> real, for the owner
|
|
19
|
+
gateway hooks reveal <file> # a run's file with the real values put back
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## The file, in one screen
|
|
23
|
+
|
|
24
|
+
```toml
|
|
25
|
+
[alias]
|
|
26
|
+
slugs = ["spycloud"] # worlds that store personal fields as sha256:<slug>:<value>
|
|
27
|
+
public_layer_wins = true # a `role: public` seed is never rewritten, even inside a private email's domain
|
|
28
|
+
public_mailbox_domains = ["gmail.com"] # kept real in an email persona; omit for the built-in free-mail list, [] for none
|
|
29
|
+
mint_inbound = ["email", "phone", "ip"] # kinds minted ON SIGHT in results; omit = seeds only
|
|
30
|
+
legend = "{run_dir}/legend.json" # the owner's way back: run key + every persona, written at run_end
|
|
31
|
+
|
|
32
|
+
[vendors]
|
|
33
|
+
spycloud = "spycloud_.*" # tool-name patterns -> a vendor label for `vendor = ...` matchers
|
|
34
|
+
|
|
35
|
+
[[hooks.pre_tool_call]] # or [[hooks.PreToolUse]] — Claude Code's names are accepted
|
|
36
|
+
matcher = "spycloud_.* | web_search" # `A | B` alternation of unanchored regexes; "" or "*" is every tool
|
|
37
|
+
handlers = [
|
|
38
|
+
{ type = "alias" }, # persona -> real on the way out
|
|
39
|
+
{ type = "command", command = "python guard.py", timeout = 10 }, # your hook: event on stdin, verdict on stdout
|
|
40
|
+
{ type = "deny", reason = "not in this run" },
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[[hooks.post_tool_call]]
|
|
44
|
+
matcher = "spycloud_.* | web_search"
|
|
45
|
+
handlers = [
|
|
46
|
+
{ type = "alias" }, # real -> persona on the way in, leaks metered
|
|
47
|
+
{ type = "record", sink = "{run_dir}/transcript.jsonl", async = true },
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[[hooks.post_tool_call]] # a SECOND rule: one rewriter per rule, and rules chain
|
|
51
|
+
matcher = "web_search | web_fetch"
|
|
52
|
+
handlers = [{ type = "redact", kinds = ["phone"], values = ["Acme Holdings"], patterns = ["\\bDOB[: ]+\\S+"] }]
|
|
53
|
+
|
|
54
|
+
[[hooks.run_end]]
|
|
55
|
+
handlers = [{ type = "alias", check = "outgoing" }] # a real value in the report fails the run
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Events: `pre_tool_call`, `post_tool_call`, `pre_request`, `post_response`, `run_end`.
|
|
59
|
+
Handlers: `alias`, `command`, `http`, `callable`, `deny`, `record`, `redact`. A rule's handlers
|
|
60
|
+
all see the same event and at most one of them may rewrite it; put a second rewriter under its
|
|
61
|
+
own `[[hooks.<event>]]` — rules apply in order and each sees the previous one's rewrite.
|
|
62
|
+
|
|
63
|
+
## The alias door
|
|
64
|
+
|
|
65
|
+
Seed it with the task's entities before the agent starts; the instruction the agent reads is
|
|
66
|
+
already in persona space, and every tool result is rewritten on the way in while every tool
|
|
67
|
+
call is resolved back to the real value on the way out.
|
|
68
|
+
|
|
69
|
+
```json
|
|
70
|
+
[{ "type": "person_name", "value": "Matthew Nowell", "identity": "subject" },
|
|
71
|
+
{ "type": "email", "value": "m.nowell@nowell-ltd.com", "identity": "subject" },
|
|
72
|
+
{ "type": "domain", "value": "star-co.net.kp", "role": "public" }]
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Kinds: `person_name`, `email`, `username`, `phone`, `company`, `domain`, `ip`, `account`,
|
|
76
|
+
`machine`, `log`; anything else gets a `REF-nnnn` stand-in. `identity` ties attributes to one
|
|
77
|
+
person so they stay consistent; `role: "public"` is never replaced. Every variant of a value is
|
|
78
|
+
tracked (bare surname, handle spellings, `(at)` emails, phone formats, percent-encoding, a
|
|
79
|
+
world's `sha256:` stored form); what slips through is counted (`leak_through_inbound`), and a
|
|
80
|
+
real value in the outgoing report fails the run (`leaks_outbound`).
|
|
81
|
+
|
|
82
|
+
## Identities that arrive, not seeded
|
|
83
|
+
|
|
84
|
+
A page or a vendor row brings people the seed list never named. By default they reach the
|
|
85
|
+
agent as they are. Two knobs change that:
|
|
86
|
+
|
|
87
|
+
- **`mint_inbound = ["email", "phone", "ip"]`** — an unseeded value of a listed kind met in a
|
|
88
|
+
result gets a persona on sight: aliased in, resolved back out on the next call, one persona
|
|
89
|
+
for the rest of the run whatever the spelling. Only these kinds are pattern-safe (a global
|
|
90
|
+
IP; a number with a `+` or an area code); a name or a handle is seeded, never guessed. The
|
|
91
|
+
whole result is read before any of it is rewritten, so a spelling in an earlier field is
|
|
92
|
+
caught by a later one. Listed in `found_inbound`.
|
|
93
|
+
- **`redact`** — removes instead of replacing, from ANY string a result carries (page body,
|
|
94
|
+
title, URL, vendor row): `kinds` for shaped values, `values` for arbitrary literals (a name,
|
|
95
|
+
a company, a phrase — any case, any whitespace), `patterns` for regular expressions;
|
|
96
|
+
`token` (default `[redacted {kind}]`) is what stands in. Runs after the door, leaves personas
|
|
97
|
+
alone, counted in `redactions` (kind and a three-letter prefix, never the value).
|
|
98
|
+
|
|
99
|
+
## Per task, not per file
|
|
100
|
+
|
|
101
|
+
What a task aliases is the task's decision — one wants a persona for every address the web
|
|
102
|
+
returns, another's selectors are public IPs that must stay facts — so the run overrides the
|
|
103
|
+
file:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
run = HookRun.from_config("gateway-hooks.toml", mint_inbound=["email"]) # this task only
|
|
107
|
+
run.seed(task["entities"] + task.get("replacements", [])) # values expected to surface, seeded early
|
|
108
|
+
run.on("post_tool_call", RedactHandler(run.alias, values=["Acme Holdings"]), matcher="web_.*")
|
|
109
|
+
run.mint("email", found_later) # one more persona, from code, any time
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
```ts
|
|
113
|
+
const run = await HookRun.fromConfig("gateway-hooks.toml", { mintInbound: [] }); // seeds only, whatever the file says
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Seeing the real data (the owner)
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
gateway hooks legend runs/<run id>/ # persona -> real, with role, provenance, identity
|
|
120
|
+
gateway hooks reveal runs/<run id>/report.md # the file with every persona put back
|
|
121
|
+
gateway hooks reveal transcript.jsonl --legend .gateway/hooks/<session>/legend.json --out transcript.real.jsonl
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The run writes the legend where `[alias] legend` says (found personas included); a coding
|
|
125
|
+
agent's state dir keeps one beside `run.json` after every event. `reveal` puts every persona
|
|
126
|
+
spelling back in a JSON or text file; a `[redacted …]` token stays — redaction removed the
|
|
127
|
+
value, and its raw copy is a `record` sink placed before the `redact` rule. The legend maps
|
|
128
|
+
personas to people: keep it with the run key, out of version control, never beside world data.
|
|
129
|
+
|
|
130
|
+
## Wiring
|
|
131
|
+
|
|
132
|
+
- **Agent SDK** — open the file (`from gatewaysdk.hooks import HookRun; run = HookRun("gateway-hooks.toml")`
|
|
133
|
+
/ `import { HookRun } from "@withgateway/sdk/hooks"; const run = await HookRun.fromConfig("gateway-hooks.toml")`),
|
|
134
|
+
then wrap the tools: Python `guarded = run.wrap({"lookup_email": lookup_email})`, TypeScript
|
|
135
|
+
`guarded = run.wrapTools({ lookup_email: lookupEmail })`; register the wrapped function wherever
|
|
136
|
+
the SDK takes a tool implementation. `run.tools()` materializes the file's `[tools.*]` (a keyed
|
|
137
|
+
web search, a connector operation) already wrapped; `verdict = run.end(report)` runs `run_end`.
|
|
138
|
+
- **Claude Code / coding agents** — `.claude/settings.json`: `PreToolUse`, `PostToolUse` and
|
|
139
|
+
`Stop` entries whose command is `gateway hooks run`. State (run key + seeds) persists under
|
|
140
|
+
`--state`, `$GATEWAY_HOOKS_STATE`, else `.gateway/hooks/<session id>`. A coding agent shows the
|
|
141
|
+
model a result before its hook runs, so the inbound rewrite cannot reach it there (the
|
|
142
|
+
command says so: `rewrite reached nobody`); route such tools through the SDK or an MCP server
|
|
143
|
+
that wraps them.
|
|
144
|
+
- **Any transport** — `run.session(session, request_fn=...)` puts `pre_request` /
|
|
145
|
+
`post_response` around a client with its own auth.
|
|
146
|
+
|
|
147
|
+
## Check before wiring
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
gateway hooks check gateway-hooks.toml --event PreToolUse --tool Bash --input '{"command":"ls"}'
|
|
151
|
+
gateway hooks check gateway-hooks.toml --seed @entities.json --tool web_search --input '{"query":"Keith Yarrow"}'
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
`check` builds every handler (a callable that does not import, a bad pattern, two rewriters on
|
|
155
|
+
one rule fail here, not mid-run) and prints what each would do to one event. `--json` adds the
|
|
156
|
+
run's meters: aliases, substitutions, leaks, found identities, redactions, denials, hook errors.
|
|
@@ -272,8 +272,9 @@ def build_default(world: str | Path, contract: Contract, *, force: bool = False)
|
|
|
272
272
|
fail(
|
|
273
273
|
"snapshot_pinned",
|
|
274
274
|
f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {digest}, which is not in the tree: "
|
|
275
|
-
f"
|
|
276
|
-
f"
|
|
275
|
+
f"run the world on the platform (`gateway worlds session open <slug>`, "
|
|
276
|
+
f"load_world_to_sandbox), which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} "
|
|
277
|
+
f"to rebuild from the data files",
|
|
277
278
|
f"/{DEFAULT_DIR}/{SNAPSHOT}",
|
|
278
279
|
)
|
|
279
280
|
return kept
|
|
@@ -356,9 +357,9 @@ def check_all(world: str | Path, contract: Contract) -> dict:
|
|
|
356
357
|
# refused: authoring edits journals and opens the world, and a served world
|
|
357
358
|
# replays the journals past a stale snapshot (WorldState opens only a
|
|
358
359
|
# current one). `stale` is in every check report so it never passes unseen.
|
|
359
|
-
found["default"]["stale"] = (
|
|
360
|
-
|
|
361
|
-
)
|
|
360
|
+
found["default"]["stale"] = pinned_default(world) is None and found["default"][
|
|
361
|
+
"batchDigest"
|
|
362
|
+
] != default_batch_digest(world)
|
|
362
363
|
for name in scenario_dirs(world):
|
|
363
364
|
snapshot = scenario_snapshot(world, name)
|
|
364
365
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
@@ -105,10 +105,20 @@ class TaskSpec:
|
|
|
105
105
|
return list(dict.fromkeys(named))
|
|
106
106
|
|
|
107
107
|
|
|
108
|
+
#: Keys a state dump carries beside its entities, which a grader reads like an
|
|
109
|
+
#: entity: ``calls`` (the session's call log) and ``snapshot``. A world cannot
|
|
110
|
+
#: declare an entity by either name (the compiler refuses it), so a check on
|
|
111
|
+
#: one is never a typo for a row entity; a seed may not target them.
|
|
112
|
+
RESERVED_STATE_KEYS = frozenset({"calls", "snapshot"})
|
|
113
|
+
|
|
114
|
+
|
|
108
115
|
def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
|
|
109
|
-
"""The entities ``spec`` names that ``defined`` (a world contract's) lacks.
|
|
116
|
+
"""The entities ``spec`` names that ``defined`` (a world contract's) lacks.
|
|
117
|
+
A grader's check on a reserved state key is not one of them."""
|
|
110
118
|
known = set(defined)
|
|
111
|
-
|
|
119
|
+
seeded = set((spec.seed or {}).get("rows", {})) if isinstance(spec.seed, dict) else set()
|
|
120
|
+
return [entity for entity in spec.entities()
|
|
121
|
+
if entity not in known and not (entity in RESERVED_STATE_KEYS and entity not in seeded)]
|
|
112
122
|
|
|
113
123
|
|
|
114
124
|
def parse_task_spec(value: Any) -> TaskSpec:
|
|
@@ -47,10 +47,16 @@ from dataclasses import dataclass, field
|
|
|
47
47
|
from pathlib import Path
|
|
48
48
|
from typing import Any
|
|
49
49
|
|
|
50
|
+
try:
|
|
51
|
+
import tomllib
|
|
52
|
+
except ModuleNotFoundError: # pragma: no cover - python 3.10
|
|
53
|
+
import tomli as tomllib
|
|
54
|
+
|
|
50
55
|
from .. import _bundle
|
|
51
56
|
from .._materialize import materialize_template
|
|
52
57
|
from .._world import World
|
|
53
58
|
from ..errors import WorldStartError
|
|
59
|
+
from ..world import World as WorldDetector
|
|
54
60
|
from . import snapshot as snapshots
|
|
55
61
|
from .api import ApiSettings, WorldState, load_api_settings
|
|
56
62
|
from .api import load_handlers as load_api_handlers
|
|
@@ -58,6 +64,7 @@ from .compiler import ContractError, bundle_mode, check_bundle, fail
|
|
|
58
64
|
from .connector import SPEC, Spec, load_spec
|
|
59
65
|
from .data import seed_batch
|
|
60
66
|
from .dependencies import Dependency, parse_dependencies
|
|
67
|
+
from .grading import world_state
|
|
61
68
|
from .tasks import TaskSpec
|
|
62
69
|
|
|
63
70
|
Handler = Callable[..., Any]
|
|
@@ -84,6 +91,12 @@ class ToolsTask:
|
|
|
84
91
|
#: verifiers; ``None`` (a task that names no worlds) grades with all of every
|
|
85
92
|
#: dependency's, the way it grades with all of the root's.
|
|
86
93
|
worlds: dict[str, tuple[str, ...]] | None = None
|
|
94
|
+
#: An authored ``scenarios/<id>/`` task: its seed overlay replaces whole
|
|
95
|
+
#: entities of the template at open and reset (None: the template alone).
|
|
96
|
+
scenario: dict[str, list] | None = None
|
|
97
|
+
#: The scenario's ``checks/verify.py``, when it grades by one instead of
|
|
98
|
+
#: the world's SQL verifiers (the runtime's :class:`TaskGrader` runs it).
|
|
99
|
+
verify: Path | None = None
|
|
87
100
|
|
|
88
101
|
def selected(self, all_verifiers: list[str]) -> list[str]:
|
|
89
102
|
"""The verifiers this task grades with: its own list, else the
|
|
@@ -106,19 +119,12 @@ INLINE_TASK_ID = "session"
|
|
|
106
119
|
|
|
107
120
|
|
|
108
121
|
def inline_task(spec: TaskSpec) -> ToolsTask:
|
|
109
|
-
"""A session task (:mod:`tasks`) as a
|
|
110
|
-
instruction as the prompt
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
:class:`ToolsWorld` once the template copy exists."""
|
|
116
|
-
if spec.grader is not None:
|
|
117
|
-
fail(
|
|
118
|
-
"task",
|
|
119
|
-
"a mock-tools world grades with its bundled verifiers; give the task no grader, or open an api world",
|
|
120
|
-
"/task/grader",
|
|
121
|
-
)
|
|
122
|
+
"""A session task (:mod:`tasks`) as a schema-world task: the spec's
|
|
123
|
+
instruction as the prompt. Without a grader of its own the task is graded
|
|
124
|
+
by every verifier the bundle ships (:meth:`ToolsWorld.grade`); with one
|
|
125
|
+
(assertions, python, rubric) the host grades it the way it grades an api
|
|
126
|
+
world's session task, over :meth:`ToolsWorld.grader_state`. The seed is
|
|
127
|
+
applied by :class:`ToolsWorld` once the template copy exists."""
|
|
122
128
|
return ToolsTask(id=spec.name or INLINE_TASK_ID, prompt=spec.instruction, metadata=dict(spec.metadata))
|
|
123
129
|
|
|
124
130
|
|
|
@@ -218,7 +224,9 @@ def served_with_dependencies(version: ToolsVersion, dependencies: dict[str, Depe
|
|
|
218
224
|
|
|
219
225
|
|
|
220
226
|
def is_tools_bundle(bundle: Path) -> bool:
|
|
221
|
-
|
|
227
|
+
"""The tree runs on the tools engine (a schema or legacy tools world stored
|
|
228
|
+
as itself; an api bundle does not)."""
|
|
229
|
+
return WorldDetector.from_dir(bundle).tools_engine
|
|
222
230
|
|
|
223
231
|
|
|
224
232
|
def verifier_names(bundle: Path) -> list[str]:
|
|
@@ -263,14 +271,37 @@ def _tasks_from_rows(rows: list) -> list[ToolsTask]:
|
|
|
263
271
|
return out
|
|
264
272
|
|
|
265
273
|
|
|
274
|
+
def _scenario_task(name: str, scenario: Path) -> ToolsTask:
|
|
275
|
+
"""A task from an authored scenario dir (what an api bundle carried and
|
|
276
|
+
``worlds migrate`` keeps): ``instruction.md`` or the toml's ``instruction``
|
|
277
|
+
is the prompt, ``seed/*.json`` the overlay, ``checks/verify.py`` the grader."""
|
|
278
|
+
table = tomllib.loads((scenario / "scenario.toml").read_text(encoding="utf-8"))
|
|
279
|
+
instruction = scenario / "instruction.md"
|
|
280
|
+
prompt = instruction.read_text(encoding="utf-8") if instruction.is_file() else str(table.get("instruction") or "")
|
|
281
|
+
verify = scenario / "checks" / "verify.py"
|
|
282
|
+
return ToolsTask(
|
|
283
|
+
id=name,
|
|
284
|
+
prompt=prompt.strip() or f"Drive the world into the state the scenario '{name}' checks.",
|
|
285
|
+
metadata={"scenario_id": name},
|
|
286
|
+
scenario=snapshots.scenario_overlay(scenario) or None,
|
|
287
|
+
verify=verify if verify.is_file() else None,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def scenario_tasks(bundle: Path) -> list[ToolsTask]:
|
|
292
|
+
"""One task per ``scenarios/<name>/scenario.toml`` of the tree."""
|
|
293
|
+
return [_scenario_task(name, path) for name, path in snapshots.scenario_dirs(bundle).items()]
|
|
294
|
+
|
|
295
|
+
|
|
266
296
|
def load_tasks(
|
|
267
297
|
bundle: Path, manifest: _bundle.EnvManifest, verifiers: list[str], *, required: bool = True
|
|
268
298
|
) -> dict[str, ToolsTask]:
|
|
269
299
|
"""The bundle's tasks, in the runner's precedence: ``[[tasks]]`` in the
|
|
270
300
|
manifest, else ``tasks.jsonl`` / ``tasks.json``, else one task per
|
|
271
|
-
verifier
|
|
272
|
-
|
|
273
|
-
|
|
301
|
+
verifier; plus one per authored ``scenarios/<name>/`` the declared tasks
|
|
302
|
+
do not already name. A bundle with none has nothing to grade and is
|
|
303
|
+
refused when ``required`` (the host opens it to grade); a local session
|
|
304
|
+
that only calls tools opens it with none."""
|
|
274
305
|
declared = manifest.raw.get("tasks")
|
|
275
306
|
tasks = _tasks_from_rows(declared) if isinstance(declared, list) else []
|
|
276
307
|
if not tasks:
|
|
@@ -296,16 +327,18 @@ def load_tasks(
|
|
|
296
327
|
)
|
|
297
328
|
for name in verifiers
|
|
298
329
|
]
|
|
299
|
-
if not tasks and required:
|
|
300
|
-
fail(
|
|
301
|
-
"host",
|
|
302
|
-
"mock-tools bundle declares no tasks and ships no verifiers/*.sql: nothing to grade",
|
|
303
|
-
)
|
|
304
330
|
by_id: dict[str, ToolsTask] = {}
|
|
305
331
|
for task in tasks:
|
|
306
332
|
if task.id in by_id:
|
|
307
333
|
fail("host", f"task id '{task.id}' is declared twice")
|
|
308
334
|
by_id[task.id] = task
|
|
335
|
+
for task in scenario_tasks(bundle):
|
|
336
|
+
by_id.setdefault(task.id, task)
|
|
337
|
+
if not by_id and required:
|
|
338
|
+
fail(
|
|
339
|
+
"host",
|
|
340
|
+
"mock-tools bundle declares no tasks and ships no verifiers/*.sql: nothing to grade",
|
|
341
|
+
)
|
|
309
342
|
return by_id
|
|
310
343
|
|
|
311
344
|
|
|
@@ -459,7 +492,10 @@ class ToolsWorld:
|
|
|
459
492
|
|
|
460
493
|
def _apply_task_seed(self) -> None:
|
|
461
494
|
"""Start the session in its task's state: the template with the task's
|
|
462
|
-
rows on top
|
|
495
|
+
rows on top — a scenario task's overlay replacing whole entities, then
|
|
496
|
+
the session task's seed. A task without either starts on the template."""
|
|
497
|
+
if self.task.scenario is not None:
|
|
498
|
+
self.seed({"rows": self.task.scenario, "mode": "replace"})
|
|
463
499
|
if self._task_seed is not None:
|
|
464
500
|
self.seed({"rows": self._task_seed["rows"], "mode": self._task_seed["mode"]})
|
|
465
501
|
|
|
@@ -606,6 +642,10 @@ class ToolsWorld:
|
|
|
606
642
|
counts[f"{prefix}.{entity}" if prefix else entity] = len(store.list(entity))
|
|
607
643
|
return {"seeded": seeded, "mode": mode, "entity_counts": counts}
|
|
608
644
|
|
|
645
|
+
@property
|
|
646
|
+
def session_dir(self) -> Path:
|
|
647
|
+
return self._dir
|
|
648
|
+
|
|
609
649
|
def state(self) -> dict:
|
|
610
650
|
"""The root's rows by entity, and under ``worlds`` each dependency's."""
|
|
611
651
|
out: dict[str, Any] = {"entities": _entities_of(self.world)}
|
|
@@ -613,6 +653,18 @@ class ToolsWorld:
|
|
|
613
653
|
out["worlds"] = {alias: {"entities": _entities_of(world)} for alias, world in self.worlds.items()}
|
|
614
654
|
return out
|
|
615
655
|
|
|
656
|
+
def entity_names(self) -> list[str]:
|
|
657
|
+
"""Every entity name :meth:`grader_state` reports, rows or not."""
|
|
658
|
+
names = list(self.world.contract.entities) if self.world.contract is not None else []
|
|
659
|
+
for alias, world in self.worlds.items():
|
|
660
|
+
if world.contract is not None:
|
|
661
|
+
names.extend(f"{alias}.{entity}" for entity in world.contract.entities)
|
|
662
|
+
return names
|
|
663
|
+
|
|
664
|
+
def grader_state(self) -> dict:
|
|
665
|
+
"""The rows as a session task's grader reads them (:func:`world_state`)."""
|
|
666
|
+
return world_state(self.world, self.worlds)
|
|
667
|
+
|
|
616
668
|
def close(self) -> None:
|
|
617
669
|
world = getattr(self, "world", None)
|
|
618
670
|
if world is not None:
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""The session protocol, once: every verb a world session answers, with the
|
|
2
|
+
method it takes and how its body is read. Both HTTP transports dispatch
|
|
3
|
+
through this table — the local session (``local.py``, ``/__session/<verb>``)
|
|
4
|
+
and the World Host control plane (``host.py``, ``/__host/sessions/<sid>/
|
|
5
|
+
<verb>``) — so the two cannot drift: a verb added here is served by both, a
|
|
6
|
+
body shape accepted here is accepted by both. The transports stay distinct
|
|
7
|
+
(loopback with no secret against a fleet with leases, workers and exports);
|
|
8
|
+
only the verbs are shared.
|
|
9
|
+
|
|
10
|
+
A verb marked ``hosted`` is the platform's rendezvous with a session it keeps
|
|
11
|
+
alive across hosts (the lease, the state export, DELETE as the platform's
|
|
12
|
+
close); the local session, which lives as long as the process serving it,
|
|
13
|
+
refuses them by name."""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from .redaction import parse_mode
|
|
22
|
+
from .validation import ContractError, fail
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class SessionVerb:
|
|
27
|
+
"""One row of the protocol: ``method`` and ``name`` are the route (``name``
|
|
28
|
+
None addresses the session itself, as ``DELETE`` does); ``read`` turns the
|
|
29
|
+
request body into the keyword arguments the session operation takes, or
|
|
30
|
+
refuses the shape (a :class:`ContractError` the transport answers 400)."""
|
|
31
|
+
|
|
32
|
+
method: str
|
|
33
|
+
name: str | None
|
|
34
|
+
read: Callable[[dict], dict[str, Any]]
|
|
35
|
+
hosted: bool = False
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def route(self) -> str:
|
|
39
|
+
return f"{self.method} {self.name}" if self.name is not None else f"{self.method} <session>"
|
|
40
|
+
|
|
41
|
+
def matches(self, method: str, name: str | None) -> bool:
|
|
42
|
+
return self.method == method and self.name == (name or None)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def read_nothing(_body: dict) -> dict[str, Any]:
|
|
46
|
+
return {}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def read_seed(body: dict) -> dict[str, Any]:
|
|
50
|
+
"""``{rows, mode, redact}``, the same under ``args`` (the platform wraps
|
|
51
|
+
the seed it forwards), or bare ``{entity: [row, ...]}`` — the shape a rows
|
|
52
|
+
file has — taken as the rows of an append."""
|
|
53
|
+
args = body["args"] if isinstance(body.get("args"), dict) else body
|
|
54
|
+
if "rows" not in args and "mode" not in args and args and all(isinstance(v, list) for v in args.values()):
|
|
55
|
+
args = {"rows": args}
|
|
56
|
+
rows = args.get("rows")
|
|
57
|
+
if not isinstance(rows, dict):
|
|
58
|
+
fail(
|
|
59
|
+
"host",
|
|
60
|
+
'seed needs rows as {"rows": {entity: [row, ...]}, "mode": "append"|"replace"}; '
|
|
61
|
+
"a dependency's entity is named alias.entity",
|
|
62
|
+
)
|
|
63
|
+
mode = args.get("mode", "append")
|
|
64
|
+
if mode not in ("append", "replace"):
|
|
65
|
+
fail("host", 'seed mode must be "append" or "replace"', "/args/mode")
|
|
66
|
+
return {"rows": rows, "mode": mode, "redact": parse_mode(args.get("redact"), "/args/redact")}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def read_grade(body: dict) -> dict[str, Any]:
|
|
70
|
+
"""``report`` is the agent's own account of the run, read by a rubric
|
|
71
|
+
judge or put beside the state for a deterministic grader; absent is fine,
|
|
72
|
+
anything but a string is not."""
|
|
73
|
+
report = body.get("report")
|
|
74
|
+
if report is not None and not isinstance(report, str):
|
|
75
|
+
fail("input", "report must be a string", "/report")
|
|
76
|
+
return {"report": report}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def read_call(body: dict) -> dict[str, Any]:
|
|
80
|
+
tool = body.get("tool")
|
|
81
|
+
if not isinstance(tool, str) or not tool:
|
|
82
|
+
fail("input", "tool must be a tool name (root) or alias.tool (a dependency's)", "/tool")
|
|
83
|
+
args = body.get("args") or {}
|
|
84
|
+
if not isinstance(args, dict):
|
|
85
|
+
fail("input", "args must be an object of the tool's arguments", "/args")
|
|
86
|
+
return {"tool": tool, "args": args}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def read_export(body: dict) -> dict[str, Any]:
|
|
90
|
+
known = body.get("knownSha256")
|
|
91
|
+
return {"known_sha256": known if isinstance(known, str) else None}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
SESSION_VERBS: tuple[SessionVerb, ...] = (
|
|
95
|
+
SessionVerb("GET", "tools", read_nothing),
|
|
96
|
+
SessionVerb("GET", "task", read_nothing),
|
|
97
|
+
SessionVerb("GET", "state", read_nothing),
|
|
98
|
+
SessionVerb("POST", "grade", read_grade),
|
|
99
|
+
SessionVerb("POST", "reset", read_nothing),
|
|
100
|
+
SessionVerb("POST", "seed", read_seed),
|
|
101
|
+
SessionVerb("POST", "call", read_call),
|
|
102
|
+
SessionVerb("POST", "close", read_nothing),
|
|
103
|
+
SessionVerb("POST", "renew", read_nothing, hosted=True),
|
|
104
|
+
SessionVerb("POST", "export", read_export, hosted=True),
|
|
105
|
+
SessionVerb("DELETE", None, read_nothing, hosted=True),
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def session_verb(method: str, name: str | None) -> SessionVerb | None:
|
|
110
|
+
return next((verb for verb in SESSION_VERBS if verb.matches(method, name)), None)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def served_verbs(*, hosted: bool) -> tuple[SessionVerb, ...]:
|
|
114
|
+
"""The rows a transport answers: every one on the host, the session's own
|
|
115
|
+
(not the lease's) locally."""
|
|
116
|
+
return tuple(verb for verb in SESSION_VERBS if hosted or not verb.hosted)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def unknown_verb(method: str, name: str | None, *, hosted: bool, prefix: str) -> dict:
|
|
120
|
+
"""The 404 body for a route off the table, naming what is on it. A hosted
|
|
121
|
+
verb sent to the local session is refused by name, not as unknown."""
|
|
122
|
+
verb = session_verb(method, name)
|
|
123
|
+
if verb is not None and verb.hosted and not hosted:
|
|
124
|
+
return {"error": f"{verb.route} is a hosted session's verb (the platform's lease); the local session has none"}
|
|
125
|
+
known = ", ".join(
|
|
126
|
+
f"{v.method} {prefix}{v.name}" if v.name else f"{v.method} {prefix.rstrip('/')}"
|
|
127
|
+
for v in served_verbs(hosted=hosted)
|
|
128
|
+
)
|
|
129
|
+
return {"error": f"unknown session route {method} {prefix}{name or ''}; one of {known}"}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def refusal(error: ContractError) -> dict:
|
|
133
|
+
"""A refused call as the answer a session gives it: an unknown tool,
|
|
134
|
+
arguments off the schema, a tool outside the task's allow-list, a write the
|
|
135
|
+
store will not take, all ``{status: 400, body: {error, detail}}``."""
|
|
136
|
+
return {"status": 400, "body": {"error": error.detail.get("code", "host"), "detail": str(error)}}
|
|
@@ -51,16 +51,25 @@ from dataclasses import dataclass, field
|
|
|
51
51
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
52
52
|
from pathlib import Path
|
|
53
53
|
|
|
54
|
+
from .grading import VerifierSandbox
|
|
55
|
+
|
|
54
56
|
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
55
57
|
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
56
|
-
#:
|
|
58
|
+
#: Memory every worker together may use, in MiB, unless
|
|
57
59
|
#: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. The number of workers a replica
|
|
58
|
-
#: runs at once is this over the
|
|
59
|
-
#:
|
|
60
|
-
#:
|
|
61
|
-
#: ``at_capacity`` and placed on another replica instead of the kernel
|
|
62
|
-
#: choosing which process to kill.
|
|
60
|
+
#: runs at once is this over the footprint each is counted at. A version past
|
|
61
|
+
#: that is refused ``at_capacity`` and placed on another replica instead of
|
|
62
|
+
#: the kernel choosing which process to kill.
|
|
63
63
|
DEFAULT_WORKER_BUDGET_MB = 4096
|
|
64
|
+
#: What one worker is counted at against the budget, in MiB, unless
|
|
65
|
+
#: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise. Unset, a worker is counted at
|
|
66
|
+
#: its whole address-space cap — two of 2 GiB inside a 6 GiB container — which
|
|
67
|
+
#: is the safe reading and a 50x over-reservation: a measured worker holding
|
|
68
|
+
#: sessions is ~40 MiB resident (prod, 2026-09-17). A fleet that must host many
|
|
69
|
+
#: worlds at once sets this to the footprint it has measured, keeps the
|
|
70
|
+
#: address-space cap as the rail each worker cannot cross, and sizes the
|
|
71
|
+
#: container for footprint x workers plus the supervisor's share.
|
|
72
|
+
DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
|
|
64
73
|
#: Largest file a worker may write (its session databases), in MiB.
|
|
65
74
|
DEFAULT_WORKER_FILE_MB = 4096
|
|
66
75
|
#: Open files per worker: one per live session plus the sockets serving them.
|
|
@@ -274,6 +283,18 @@ def confined_command(script: Path, python: str = sys.executable) -> list[str]:
|
|
|
274
283
|
return [python, "-c", _CONFINED_PRELUDE, str(script)]
|
|
275
284
|
|
|
276
285
|
|
|
286
|
+
class ConfinedVerifierSandbox(VerifierSandbox):
|
|
287
|
+
"""How the host runs a task's ``verify.py``: confined like a worker, with a
|
|
288
|
+
scrubbed environment and a home under the session. World-authored code
|
|
289
|
+
sees no secret of this process, be it the host's or a worker's."""
|
|
290
|
+
|
|
291
|
+
def command(self, script: Path) -> list[str]:
|
|
292
|
+
return confined_command(script)
|
|
293
|
+
|
|
294
|
+
def environment(self, home: Path) -> dict[str, str]:
|
|
295
|
+
return verifier_environment(home=home)
|
|
296
|
+
|
|
297
|
+
|
|
277
298
|
# --------------------------------------------------------------------------- #
|
|
278
299
|
# The supervisor's side: start, reach and stop workers.
|
|
279
300
|
# --------------------------------------------------------------------------- #
|
|
@@ -348,13 +369,22 @@ class WorkerPool:
|
|
|
348
369
|
worlds_domain: str,
|
|
349
370
|
memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
|
|
350
371
|
budget_mb: int = DEFAULT_WORKER_BUDGET_MB,
|
|
372
|
+
footprint_mb: int | None = DEFAULT_WORKER_FOOTPRINT_MB,
|
|
351
373
|
idle_s: float = DEFAULT_WORKER_IDLE_S,
|
|
352
374
|
python: str = sys.executable,
|
|
353
375
|
):
|
|
354
|
-
if
|
|
376
|
+
if footprint_mb is None:
|
|
377
|
+
footprint_mb = memory_mb
|
|
378
|
+
if footprint_mb <= 0 or footprint_mb > memory_mb:
|
|
379
|
+
raise ValueError(
|
|
380
|
+
f"a worker's footprint ({footprint_mb} MiB) must be between 1 MiB and its "
|
|
381
|
+
f"address-space cap ({memory_mb} MiB) (WORLDS_WORKER_FOOTPRINT_MB, "
|
|
382
|
+
"WORLDS_WORKER_MEMORY_MB)"
|
|
383
|
+
)
|
|
384
|
+
if budget_mb < footprint_mb:
|
|
355
385
|
raise ValueError(
|
|
356
|
-
f"the worker budget ({budget_mb} MiB) holds no worker of {
|
|
357
|
-
"(WORLDS_WORKER_BUDGET_MB,
|
|
386
|
+
f"the worker budget ({budget_mb} MiB) holds no worker of {footprint_mb} MiB "
|
|
387
|
+
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
|
|
358
388
|
)
|
|
359
389
|
self.root = Path(cache_root)
|
|
360
390
|
self.workers_root = self.root / "workers"
|
|
@@ -365,8 +395,9 @@ class WorkerPool:
|
|
|
365
395
|
self.worlds_domain = worlds_domain
|
|
366
396
|
self.memory_mb = memory_mb
|
|
367
397
|
self.budget_mb = budget_mb
|
|
368
|
-
|
|
369
|
-
|
|
398
|
+
self.footprint_mb = footprint_mb
|
|
399
|
+
#: Workers live at once: the budget's worth of footprints.
|
|
400
|
+
self.max_workers = budget_mb // footprint_mb
|
|
370
401
|
self.idle_s = idle_s
|
|
371
402
|
self.python = python
|
|
372
403
|
self._lock = threading.Lock()
|
|
@@ -403,8 +434,8 @@ class WorkerPool:
|
|
|
403
434
|
self._workers.pop(spare.pop(0).version_id).stop()
|
|
404
435
|
if len(self._workers) >= self.max_workers:
|
|
405
436
|
raise WorkerCapacity(
|
|
406
|
-
f"this host's {self.max_workers} version workers ({self.budget_mb} MiB
|
|
407
|
-
f"{self.
|
|
437
|
+
f"this host's {self.max_workers} version workers ({self.budget_mb} MiB at "
|
|
438
|
+
f"{self.footprint_mb} MiB each) all hold sessions; place elsewhere"
|
|
408
439
|
)
|
|
409
440
|
|
|
410
441
|
def peek(self, version_id: str) -> Worker | None:
|