@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# A world from nothing: `sandbox open --new`
|
|
2
|
+
|
|
3
|
+
`gateway worlds sandbox open --new <slug> --from custom|<template>` (`bench sandbox` is the
|
|
4
|
+
same command; MCP `open_world_sandbox({ new, from })`) opens a hosted sandbox for a world that
|
|
5
|
+
does not exist yet in the key's project. Nothing is needed on the caller's disk — not a
|
|
6
|
+
checkout, not `schema init`.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
gateway worlds sandbox open --new acme-crm --from custom # or --from slack|github|…
|
|
10
|
+
gateway worlds sandbox exec <id> "gateway worlds schema entity add . crm_accounts --fields id:string,name:string --key id"
|
|
11
|
+
gateway worlds sandbox exec <id> "gateway worlds schema compile . && gateway worlds schema check ."
|
|
12
|
+
gateway worlds sandbox check <id> # the push gate the first commit will run
|
|
13
|
+
gateway worlds sandbox commit <id> -m "first cut" # creates the world: first READY version on main
|
|
14
|
+
gateway worlds sandbox close <id>
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
What `open` does and does not do:
|
|
18
|
+
|
|
19
|
+
- The slug goes through the one world slug validator and must be free in the project. A taken
|
|
20
|
+
slug is a 409 `world_exists`; the sandbox never renames it (pick another slug).
|
|
21
|
+
- `--from custom` seeds the working tree with exactly what `gateway worlds schema init <slug>`
|
|
22
|
+
writes; `--from <template>` with a shipped template (`gateway worlds schema templates`,
|
|
23
|
+
`GET /api/public/worlds/templates`). An unknown template is a 400 that names the list.
|
|
24
|
+
- No world, container or version is created at open. `open`/`status` say so:
|
|
25
|
+
`containerId: null`, `baseVersionId: null`, `baseContentHash: null`, `branch: "main"`,
|
|
26
|
+
`pendingSlug: "<slug>"`.
|
|
27
|
+
- Paths stay under `/workspace/env`; the staged `gateway` CLI and `sqlite3` run on the tree.
|
|
28
|
+
|
|
29
|
+
The first `commit`:
|
|
30
|
+
|
|
31
|
+
- runs the same push gate as `check` (nothing is written to the versions tables when it fails);
|
|
32
|
+
- creates the world through the door `gateway worlds create --from <dir>` uses — slug, kind
|
|
33
|
+
detected from the tree (a retired kind is refused the same way), first READY version on
|
|
34
|
+
`main`, attributed to the key;
|
|
35
|
+
- flips the sandbox to a normal one in the same transaction: it now has a container, a base
|
|
36
|
+
version and a base hash, and every later `commit` behaves like today's (parent check, dedup,
|
|
37
|
+
no-change).
|
|
38
|
+
|
|
39
|
+
Two new-world sandboxes racing for one slug: exactly one first commit wins; the other gets a
|
|
40
|
+
409 `world_exists` and keeps its tree intact. To keep that work, open a new sandbox with another
|
|
41
|
+
slug (`sandbox open --new <other> --from custom`), copy the files across with `read`/`write`,
|
|
42
|
+
then `sandbox close <id> --force` the loser. A never-committed `--new` sandbox counts as
|
|
43
|
+
uncommitted (its base is empty), so a plain `close` exits 1; a closed sandbox cannot be reopened.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# From an OpenAPI spec: `schema init --from-openapi`
|
|
2
|
+
|
|
3
|
+
`gateway worlds schema init <dir> --from-openapi <source>` turns a Swagger 2.0 or OpenAPI 3.0.x /
|
|
4
|
+
3.1 document (JSON or YAML) into a world: `schema/world.json` (entities), `connector.toml` (the
|
|
5
|
+
served routes) and `schema/provenance.json` (where it came from). The importer is the Python
|
|
6
|
+
package's `OpenApiImport` → `OpenApiWorldPlan`; the plan is handed to the same scaffold writer
|
|
7
|
+
`schema init` always uses, so the tree it writes is a plain schema world you keep editing with
|
|
8
|
+
`schema entity add` and `connector route add`.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
gateway worlds schema init ./x --from-openapi petstore.yaml # local file: plan printed, tree written
|
|
12
|
+
gateway worlds schema init ./x --from-openapi drive:specs/petstore.json # a file on the project drive (read platform-side, never downloaded)
|
|
13
|
+
gateway worlds schema init ./x --from-openapi https://vendor.test/openapi.json # fetched from where the command runs
|
|
14
|
+
gateway worlds schema init ./x --from-openapi github.json --plan-only # print the plan, write nothing
|
|
15
|
+
gateway worlds schema init ./x --from-openapi github.json --include '/repos/{owner}/{repo}/issues*' --exclude reactions
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Sources
|
|
19
|
+
|
|
20
|
+
| `<source>` | Read by | Notes |
|
|
21
|
+
| --- | --- | --- |
|
|
22
|
+
| a local path | the CLI process | must be a file under 64 MiB |
|
|
23
|
+
| `drive:<path>` | the platform, with your `GATEWAY_*` keys | the same reader as `worlds data import drive:…`; bytes stay in memory, nothing lands on disk |
|
|
24
|
+
| `http(s)://…` | the process running the command | the TS CLI fetches it like `files fetch`; the Python `gateway` fetches behind its private-address guard (loopback, RFC 1918, link-local and metadata addresses are refused, redirects re-checked). Other schemes (`file:`, `ftp:`) are refused. |
|
|
25
|
+
|
|
26
|
+
Inside a world sandbox (`run_gateway_cli`, `sandbox exec`) only a workspace path is accepted:
|
|
27
|
+
write the document into the workspace first (`sandbox write --from spec.yaml`), then import.
|
|
28
|
+
|
|
29
|
+
## What the plan says
|
|
30
|
+
|
|
31
|
+
`--from-openapi` always prints the plan first (JSON, one object):
|
|
32
|
+
|
|
33
|
+
```json
|
|
34
|
+
{
|
|
35
|
+
"status": "ok", "written": true,
|
|
36
|
+
"plan": {
|
|
37
|
+
"source": "petstore.yaml", "document": {"version": "openapi 3.0.3", "title": "Petstore", "sha256": "…"},
|
|
38
|
+
"connector": "petstore",
|
|
39
|
+
"entities": [{"name": "pets", "key": "id", "source": "Pet", "pointer": "/components/schemas/Pet"}],
|
|
40
|
+
"operations": [{"name": "list_pets", "method": "GET", "path": "/pets", "entity": "pets", "action": "read",
|
|
41
|
+
"params": ["limit", "status"], "filter": {"status": "status"}, "pointer": "/paths/~1pets/get"}],
|
|
42
|
+
"skipped": [{"method": "TRACE", "path": "/pets", "pointer": "/paths/~1pets/trace", "skipped_reason": "method TRACE is not one a connector serves"}],
|
|
43
|
+
"excluded": 0, "warnings": ["/paths/~1pets/post: operation name add_pet is taken; using add_pet_2"]
|
|
44
|
+
},
|
|
45
|
+
"domain": "petstore", "next": ["gateway worlds schema compile ./x # …", "…"]
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
With `--plan-only` the same object carries `"written": false` and no `domain`/`next`.
|
|
50
|
+
|
|
51
|
+
## Conversion rules (decided; the importer documents them in `schema/provenance.json`)
|
|
52
|
+
|
|
53
|
+
- **Entities.** Every object schema under `definitions` / `components.schemas` that some selected
|
|
54
|
+
operation returns and that has an id-like key (`id`, `<name>Id`, `<name>_id`, a `readOnly`
|
|
55
|
+
field the vendor marks `x-unique`/`x-primary-key`, else the first `required` string/integer field) becomes a `$defs` entry plus
|
|
56
|
+
`x-gateway.entities.<snake_plural>` with `primary_key` (snake_case, at most 64 characters —
|
|
57
|
+
a longer vendor name is truncated, the original kept in provenance). Keyless objects stay `$defs`,
|
|
58
|
+
referenced inline. `allOf` is merged; `oneOf`/`anyOf` become a union the world validator
|
|
59
|
+
accepts. Entity roots close (`additionalProperties: false`); the vendor's field names are kept
|
|
60
|
+
as spelled (so a rows file shaped like the vendor's response imports as it is); the vendor's
|
|
61
|
+
schema name stays the `$defs` key and `schema/provenance.json` maps every entity and operation
|
|
62
|
+
back to its source name and JSON pointer.
|
|
63
|
+
- **Routes.** Every path+method in `get|post|put|patch|delete` becomes an `[[operations]]` entry:
|
|
64
|
+
`name` = `operationId` snake_cased, at most 64 characters, deduped with `_2`, `_3` and a
|
|
65
|
+
warning; `entity` = the response schema's entity when it resolves to one (an array → a list
|
|
66
|
+
route, an object → `read|create|update|delete` by method); path parameters and declared query
|
|
67
|
+
parameters become `params`; a `{data: [...]}` wrapper becomes `results`; on a list route a
|
|
68
|
+
declared query parameter that names a top-level entity field becomes a `filter` (paging names
|
|
69
|
+
such as `page`, `limit`, `offset`, `cursor` never do). A path that repeats a placeholder
|
|
70
|
+
(`/a/{id}/{id}`) is skipped with its reason. An operation whose response is not an
|
|
71
|
+
entity (login, upload, HTML) stays a **passthrough** route with no entity — it is served, and
|
|
72
|
+
answers `501 {"error": "not_modelled"}` until you give it a handler.
|
|
73
|
+
- **Skipped, with a reason:** `HEAD`/`OPTIONS`/`TRACE`; a path argument that is not an
|
|
74
|
+
identifier; a second operation on the same method+path shape. **Refused, with a JSON pointer
|
|
75
|
+
into the document, nothing written:** no `paths`; an unknown or external `$ref`; an entity root
|
|
76
|
+
that is (transitively) cyclic — model the recursion as a relationship field (`parent_id`)
|
|
77
|
+
instead; a path parameter nothing declares (`--param <name>` on a hand-written route); a
|
|
78
|
+
document with no keyed schema at all.
|
|
79
|
+
- **Big documents.** `--include GLOB|TAG` / `--exclude GLOB|TAG` (repeatable) select operations
|
|
80
|
+
by path glob or tag before anything is resolved; only the schemas the selected operations reach
|
|
81
|
+
are converted, so the 9.9 MB GitHub document imports as an issues subset in seconds. A
|
|
82
|
+
document over 64 MiB is refused before it is read.
|
|
83
|
+
|
|
84
|
+
## After import
|
|
85
|
+
|
|
86
|
+
The printed `next` steps are the loop, in order: `schema compile`, `schema check`, `data import
|
|
87
|
+
<rows.json> --entity <first list entity>` (rows shaped like the vendor's response), `worlds serve`
|
|
88
|
+
then `worlds call http://127.0.0.1:<port> <list_route> --args '{}'`, and `sandbox commit` when the
|
|
89
|
+
tree lives in a sandbox. `connector conform ./x --openapi <the same document>` then checks the
|
|
90
|
+
served world back against the spec.
|
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import sqlite3
|
|
7
|
+
import sys
|
|
7
8
|
import threading
|
|
8
9
|
import uuid
|
|
9
10
|
from contextlib import contextmanager
|
|
@@ -234,12 +235,7 @@ class SQLiteStore:
|
|
|
234
235
|
)
|
|
235
236
|
self._conn.row_factory = sqlite3.Row
|
|
236
237
|
try:
|
|
237
|
-
|
|
238
|
-
# a string literal, so `WHERE "status" = 'ACTIVE'` against a table
|
|
239
|
-
# with no status column compares two literals and scores 0 without
|
|
240
|
-
# a word (Brex retrospective). Refuse the fallback: name a column
|
|
241
|
-
# that does not exist and the statement fails, saying which.
|
|
242
|
-
self._conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
|
|
238
|
+
self.forbid_double_quoted_literals(self._conn)
|
|
243
239
|
self._conn.execute("PRAGMA foreign_keys = ON")
|
|
244
240
|
if self._conn.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
|
|
245
241
|
fail("sqlite", "Foreign-key enforcement is required")
|
|
@@ -260,6 +256,19 @@ class SQLiteStore:
|
|
|
260
256
|
self.path.unlink(missing_ok=True)
|
|
261
257
|
raise
|
|
262
258
|
|
|
259
|
+
@staticmethod
|
|
260
|
+
def forbid_double_quoted_literals(conn: Any) -> None:
|
|
261
|
+
"""SQLite reads a double-quoted name that is not a column as a string literal, so a
|
|
262
|
+
verifier naming a missing column compares two literals and scores 0 without a word.
|
|
263
|
+
Switch the fallback off, or refuse to open where Python cannot (``setconfig`` is 3.12+)."""
|
|
264
|
+
if not hasattr(conn, "setconfig"):
|
|
265
|
+
fail(
|
|
266
|
+
"python_version",
|
|
267
|
+
f"Python {sys.version_info.major}.{sys.version_info.minor} cannot turn off SQLite's "
|
|
268
|
+
"double-quoted string literals; the world store needs Python 3.12 or newer.",
|
|
269
|
+
)
|
|
270
|
+
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
|
|
271
|
+
|
|
263
272
|
@classmethod
|
|
264
273
|
def create(cls, path: str | Path, contract: Contract) -> "SQLiteStore":
|
|
265
274
|
return cls(Path(path), contract, create=True)
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""THE ``store`` A HANDLER MAY CALL — one list, read by every surface.
|
|
2
|
+
|
|
3
|
+
A custom tool in ``tools/handlers.py`` reaches the world's rows through
|
|
4
|
+
``from gateway_env import store``. That object is a facade over the schema
|
|
5
|
+
store (:class:`~gatewaysdk.environment.schema.store.SQLiteStore`) exposing
|
|
6
|
+
exactly the methods named here — nothing else, so a handler can never reach
|
|
7
|
+
the store's own transaction, snapshot or raw-SQL machinery.
|
|
8
|
+
|
|
9
|
+
This module is the single source of the contract: the runtime facade admits
|
|
10
|
+
these names, ``schema check`` flags any other ``store.<name>`` statically,
|
|
11
|
+
``schema init`` writes them into the scaffolded handler docstring, and a test
|
|
12
|
+
holds the skills and CLI help to the same list. ``describe_store`` reads the
|
|
13
|
+
signature off the real class, so the printed contract cannot drift from it.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import inspect
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
|
|
21
|
+
from .store import SQLiteStore
|
|
22
|
+
|
|
23
|
+
#: Every public method of the store class by name, read once with `inspect`
|
|
24
|
+
#: so a contract entry resolves through an explicit table, not attribute lookup.
|
|
25
|
+
_STORE_MEMBERS: dict[str, object] = {
|
|
26
|
+
name: member
|
|
27
|
+
for name, member in inspect.getmembers(SQLiteStore, inspect.isfunction)
|
|
28
|
+
if not name.startswith("_")
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class StoreMethod:
|
|
34
|
+
"""One handler-facing store method: its name and one line of semantics."""
|
|
35
|
+
|
|
36
|
+
name: str
|
|
37
|
+
semantics: str
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def signature(self) -> str:
|
|
41
|
+
"""``name(entity, key) -> dict | None`` — read off :class:`SQLiteStore`."""
|
|
42
|
+
sig = inspect.signature(_STORE_MEMBERS[self.name])
|
|
43
|
+
params = [p for p in sig.parameters.values() if p.name != "self"]
|
|
44
|
+
# The store is written under `from __future__ import annotations`, so
|
|
45
|
+
# every annotation is a string; the quotes are not part of the type.
|
|
46
|
+
rendered = str(sig.replace(parameters=params)).replace("'", "")
|
|
47
|
+
return f"{self.name}{rendered}"
|
|
48
|
+
|
|
49
|
+
def line(self) -> str:
|
|
50
|
+
return f"{self.signature}\n {self.semantics}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
#: The contract, in the order the docs list it. A name here MUST be a public
|
|
54
|
+
#: method of ``SQLiteStore`` (tested); a method of the store that is not here
|
|
55
|
+
#: is not a handler's to call.
|
|
56
|
+
STORE_METHODS: tuple[StoreMethod, ...] = (
|
|
57
|
+
StoreMethod(
|
|
58
|
+
"get",
|
|
59
|
+
"One row of `entity` by its primary key, or None when no row has that key.",
|
|
60
|
+
),
|
|
61
|
+
StoreMethod("list", "Every row of `entity`, ordered by key."),
|
|
62
|
+
StoreMethod(
|
|
63
|
+
"list_where",
|
|
64
|
+
"A page of `entity` rows whose every `where` field holds one of the listed values, plus the total match count.",
|
|
65
|
+
),
|
|
66
|
+
StoreMethod(
|
|
67
|
+
"insert",
|
|
68
|
+
"Validate `value` against the entity's schema and add it; the key must be new.",
|
|
69
|
+
),
|
|
70
|
+
StoreMethod(
|
|
71
|
+
"update",
|
|
72
|
+
"Merge `patch` into the row at `key` and validate the result; returns the row.",
|
|
73
|
+
),
|
|
74
|
+
StoreMethod(
|
|
75
|
+
"delete", "Remove the row at `key`; a missing row is a contract error."
|
|
76
|
+
),
|
|
77
|
+
StoreMethod(
|
|
78
|
+
"apply_batch",
|
|
79
|
+
"Apply one ingest batch {id, target, mapping, coverage, mutations} atomically; an exact retry is a no-op.",
|
|
80
|
+
),
|
|
81
|
+
StoreMethod(
|
|
82
|
+
"lineage",
|
|
83
|
+
"The batches that wrote the row at `key`, oldest first, with each batch's source.",
|
|
84
|
+
),
|
|
85
|
+
StoreMethod("counts", "Rows per entity, as {entity: count}."),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
STORE_METHOD_NAMES: tuple[str, ...] = tuple(method.name for method in STORE_METHODS)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def store_method_list() -> str:
|
|
92
|
+
"""``get, list, list_where, …`` — the form every refusal names."""
|
|
93
|
+
return ", ".join(STORE_METHOD_NAMES)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def describe_store(indent: str = "") -> str:
|
|
97
|
+
"""The contract as documentation: one signature line and one semantics line
|
|
98
|
+
per method, each prefixed with ``indent``."""
|
|
99
|
+
lines: list[str] = []
|
|
100
|
+
for method in STORE_METHODS:
|
|
101
|
+
lines.append(f"{indent}{method.signature}")
|
|
102
|
+
lines.append(f"{indent} {method.semantics}")
|
|
103
|
+
return "\n".join(lines)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def unknown_store_method_detail(name: str) -> str:
|
|
107
|
+
"""The `detail` of a refusal for ``store.<name>`` that is not in the contract."""
|
|
108
|
+
return f"store has no method '{name}'; available: {store_method_list()}"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def schema_check_help() -> str:
|
|
112
|
+
"""The `schema check` command's help, naming the store API and the static handler pass."""
|
|
113
|
+
return (
|
|
114
|
+
"Check a world bundle: compile the contract, read every tools/*.py statically, then (with the bundled "
|
|
115
|
+
"runtime) import the handlers and open one session. Handlers reach the world's rows through "
|
|
116
|
+
f"`from gateway_env import store`, the store API: {store_method_list()}. Any other `store.<name>` is "
|
|
117
|
+
"refused at file:line with that list; a hosted call into such a handler would answer 400 "
|
|
118
|
+
'{error: "handler", detail, handler}. Nothing is written.'
|
|
119
|
+
)
|
|
@@ -42,6 +42,8 @@ RUBRIC_CRITERIA = 50
|
|
|
42
42
|
RUBRIC_CRITERION_BYTES = 2 * 1024
|
|
43
43
|
TOOLS = 200
|
|
44
44
|
TOOL_NAME_CHARS = 200
|
|
45
|
+
#: Cross-world verifiers one task may carry.
|
|
46
|
+
VERIFIERS = 32
|
|
45
47
|
|
|
46
48
|
SEED_MODES = ("append", "replace")
|
|
47
49
|
ASSERTIONS = ("exists", "absent", "count", "all", "any")
|
|
@@ -53,6 +55,8 @@ _TOOL_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]*$")
|
|
|
53
55
|
|
|
54
56
|
TASK_DIR = "task"
|
|
55
57
|
TASK_FILE = "task.json"
|
|
58
|
+
#: Under the task dir, a python verifier's source: ``verifiers/<name>/verify.py``.
|
|
59
|
+
VERIFIERS_DIR = "verifiers"
|
|
56
60
|
|
|
57
61
|
|
|
58
62
|
def _bytes(value: Any) -> int:
|
|
@@ -78,6 +82,9 @@ class TaskSpec:
|
|
|
78
82
|
raw: dict
|
|
79
83
|
#: The tools the session may call, or None for every tool the world serves.
|
|
80
84
|
tools: tuple[str, ...] | None = None
|
|
85
|
+
#: Cross-world verifiers anchored on this session: each a grader table
|
|
86
|
+
#: with ``name`` and ``scope`` (see ``TaskGrader.evidence``).
|
|
87
|
+
verifiers: tuple[dict, ...] = ()
|
|
81
88
|
|
|
82
89
|
@property
|
|
83
90
|
def grader_kind(self) -> str | None:
|
|
@@ -102,6 +109,15 @@ class TaskSpec:
|
|
|
102
109
|
named.extend(self.seed["rows"])
|
|
103
110
|
if self.grader is not None and self.grader["kind"] == "assertions":
|
|
104
111
|
named.extend(check["entity"] for check in self.grader["checks"])
|
|
112
|
+
# A verifier's checks name other worlds' entities as well; only the
|
|
113
|
+
# ones under this session's own alias are this world's to define.
|
|
114
|
+
own = f"{self.name}."
|
|
115
|
+
for verifier in self.verifiers:
|
|
116
|
+
named.extend(
|
|
117
|
+
check["entity"][len(own) :]
|
|
118
|
+
for check in verifier.get("checks", ())
|
|
119
|
+
if check["entity"].startswith(own)
|
|
120
|
+
)
|
|
105
121
|
return list(dict.fromkeys(named))
|
|
106
122
|
|
|
107
123
|
|
|
@@ -125,7 +141,7 @@ def parse_task_spec(value: Any) -> TaskSpec:
|
|
|
125
141
|
"""The spec checked for shape and size; every refusal names its path."""
|
|
126
142
|
if not isinstance(value, dict):
|
|
127
143
|
fail("task", "a task is a json object", "/task")
|
|
128
|
-
unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata"}
|
|
144
|
+
unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata", "verifiers"}
|
|
129
145
|
if unknown:
|
|
130
146
|
fail("task", f"unknown task field(s): {', '.join(sorted(unknown))}", "/task")
|
|
131
147
|
name = value.get("name")
|
|
@@ -139,6 +155,9 @@ def parse_task_spec(value: Any) -> TaskSpec:
|
|
|
139
155
|
seed = _parse_seed(value.get("seed"))
|
|
140
156
|
grader = _parse_grader(value.get("grader"))
|
|
141
157
|
tools = _parse_tools(value.get("tools"))
|
|
158
|
+
verifiers = _parse_verifiers(value.get("verifiers"), name)
|
|
159
|
+
if verifiers and grader is not None and grader["kind"] == "rubric":
|
|
160
|
+
fail("task", "a task with verifiers grades by assertions, python or nothing: a rubric never reaches the host's grade", "/task/grader/kind")
|
|
142
161
|
metadata = value.get("metadata")
|
|
143
162
|
if metadata is None:
|
|
144
163
|
metadata = {}
|
|
@@ -157,11 +176,75 @@ def parse_task_spec(value: Any) -> TaskSpec:
|
|
|
157
176
|
raw["tools"] = list(tools)
|
|
158
177
|
if metadata:
|
|
159
178
|
raw["metadata"] = metadata
|
|
179
|
+
if verifiers:
|
|
180
|
+
raw["verifiers"] = list(verifiers)
|
|
160
181
|
return TaskSpec(
|
|
161
|
-
instruction=instruction,
|
|
182
|
+
instruction=instruction,
|
|
183
|
+
name=name,
|
|
184
|
+
seed=seed,
|
|
185
|
+
grader=grader,
|
|
186
|
+
metadata=metadata,
|
|
187
|
+
raw=raw,
|
|
188
|
+
tools=tools,
|
|
189
|
+
verifiers=verifiers,
|
|
162
190
|
)
|
|
163
191
|
|
|
164
192
|
|
|
193
|
+
_VERIFIER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _parse_verifiers(value: Any, task_name: str | None) -> tuple[dict, ...]:
|
|
197
|
+
"""``verifiers``: cross-world graders, each ``{name, scope, ...grader}``
|
|
198
|
+
(the grader through :func:`_parse_grader`, so the check grammar and the
|
|
199
|
+
caps are defined once). A scope names two or more distinct aliases; the
|
|
200
|
+
first is this session, so it must be the task's own name — a session task
|
|
201
|
+
is what one alias of a multi-world task opens on, named for that alias.
|
|
202
|
+
Every assertions check names ``<alias>.<entity>`` with the alias in
|
|
203
|
+
scope. Mirrors ``WorldTaskVerifierSchema``."""
|
|
204
|
+
if value is None:
|
|
205
|
+
return ()
|
|
206
|
+
at = "/task/verifiers"
|
|
207
|
+
if not isinstance(value, list) or not value:
|
|
208
|
+
fail("task", "verifiers is a non-empty list", at)
|
|
209
|
+
if len(value) > VERIFIERS:
|
|
210
|
+
fail("task", f"a task carries at most {VERIFIERS} verifiers", at)
|
|
211
|
+
parsed = [_parse_verifier(row, f"{at}/{i}", task_name) for i, row in enumerate(value)]
|
|
212
|
+
if len({row["name"] for row in parsed}) != len(parsed):
|
|
213
|
+
fail("task", "a verifier name is used once", at)
|
|
214
|
+
return tuple(parsed)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _parse_verifier(row: Any, at: str, task_name: str | None) -> dict:
|
|
218
|
+
if not isinstance(row, dict):
|
|
219
|
+
fail("task", "a verifier is a json object", at)
|
|
220
|
+
name = row.get("name")
|
|
221
|
+
if not isinstance(name, str) or not _VERIFIER_NAME.match(name) or len(name) > NAME_CHARS:
|
|
222
|
+
fail("task", "a verifier name is letters, digits, _ . -", f"{at}/name")
|
|
223
|
+
scope = row.get("scope")
|
|
224
|
+
if not isinstance(scope, list) or len(scope) < 2 or any(not isinstance(a, str) or not _NAME.match(a) for a in scope):
|
|
225
|
+
fail("task", "a verifier's scope names two or more world aliases", f"{at}/scope")
|
|
226
|
+
if len(set(scope)) != len(scope):
|
|
227
|
+
fail("task", "a verifier's scope lists each alias once", f"{at}/scope")
|
|
228
|
+
if scope[0] != task_name:
|
|
229
|
+
fail("task", f"verifier '{name}' runs on the session of its first scope alias, so the task's name must be '{scope[0]}'", f"{at}/scope")
|
|
230
|
+
grader = _parse_grader({k: v for k, v in row.items() if k not in ("name", "scope")}, at)
|
|
231
|
+
if grader is None or grader["kind"] == "rubric":
|
|
232
|
+
fail("task", "a verifier grades by assertions or python (a rubric judge reads one world)", f"{at}/kind")
|
|
233
|
+
for j, check in enumerate(grader.get("checks", ())):
|
|
234
|
+
if check["entity"].split(".", 1)[0] not in scope or "." not in check["entity"]:
|
|
235
|
+
fail(
|
|
236
|
+
"task",
|
|
237
|
+
f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
|
|
238
|
+
f"{at}/checks/{j}/entity",
|
|
239
|
+
)
|
|
240
|
+
return {"name": name, "scope": list(scope), **grader}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def verifier_source(task_dir: Path, name: str) -> Path:
|
|
244
|
+
"""Where :func:`write_task_dir` puts a python verifier's ``verify.py``."""
|
|
245
|
+
return Path(task_dir) / VERIFIERS_DIR / name / "verify.py"
|
|
246
|
+
|
|
247
|
+
|
|
165
248
|
def _parse_tools(tools: Any) -> tuple[str, ...] | None:
|
|
166
249
|
if tools is None:
|
|
167
250
|
return None
|
|
@@ -217,20 +300,20 @@ def _parse_seed(seed: Any) -> dict | None:
|
|
|
217
300
|
return {"mode": mode, "rows": rows, **({"redact": redact} if redact != "off" else {})}
|
|
218
301
|
|
|
219
302
|
|
|
220
|
-
def _parse_grader(grader: Any) -> dict | None:
|
|
303
|
+
def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
221
304
|
if grader is None:
|
|
222
305
|
return None
|
|
223
306
|
if not isinstance(grader, dict):
|
|
224
|
-
fail("task", "grader is a json object with a kind",
|
|
307
|
+
fail("task", "grader is a json object with a kind", at)
|
|
225
308
|
kind = grader.get("kind")
|
|
226
309
|
if kind == "python":
|
|
227
310
|
if set(grader) - {"kind", "source", "timeoutSeconds"}:
|
|
228
|
-
fail("task", "a python grader is {kind, source, timeoutSeconds?}",
|
|
311
|
+
fail("task", "a python grader is {kind, source, timeoutSeconds?}", at)
|
|
229
312
|
source = grader.get("source")
|
|
230
313
|
if not isinstance(source, str) or not source:
|
|
231
|
-
fail("task", "a python grader needs its verify.py source", "/
|
|
314
|
+
fail("task", "a python grader needs its verify.py source", f"{at}/source")
|
|
232
315
|
if _bytes(source) > PYTHON_SOURCE_BYTES:
|
|
233
|
-
fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", "/
|
|
316
|
+
fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
|
|
234
317
|
out: dict = {"kind": "python", "source": source}
|
|
235
318
|
timeout = grader.get("timeoutSeconds")
|
|
236
319
|
if timeout is not None:
|
|
@@ -239,18 +322,18 @@ def _parse_grader(grader: Any) -> dict | None:
|
|
|
239
322
|
or not isinstance(timeout, (int, float))
|
|
240
323
|
or not 0 < timeout <= PYTHON_TIMEOUT_S
|
|
241
324
|
):
|
|
242
|
-
fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", "/
|
|
325
|
+
fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
|
|
243
326
|
out["timeoutSeconds"] = float(timeout)
|
|
244
327
|
return out
|
|
245
328
|
if kind == "assertions":
|
|
246
329
|
if set(grader) - {"kind", "checks"}:
|
|
247
|
-
fail("task", "an assertions grader is {kind, checks}",
|
|
330
|
+
fail("task", "an assertions grader is {kind, checks}", at)
|
|
248
331
|
checks = grader.get("checks")
|
|
249
332
|
if not isinstance(checks, list) or not checks:
|
|
250
|
-
fail("task", "an assertions grader needs at least one check", "/
|
|
333
|
+
fail("task", "an assertions grader needs at least one check", f"{at}/checks")
|
|
251
334
|
if len(checks) > CHECKS:
|
|
252
|
-
fail("task", f"an assertions grader has at most {CHECKS} checks", "/
|
|
253
|
-
parsed = [_parse_check(check, f"/
|
|
335
|
+
fail("task", f"an assertions grader has at most {CHECKS} checks", f"{at}/checks")
|
|
336
|
+
parsed = [_parse_check(check, f"{at}/checks/{i}") for i, check in enumerate(checks)]
|
|
254
337
|
# Every check owns one reward key, its id or its position; the keys are
|
|
255
338
|
# distinct as a set, so an explicit `check_2` cannot shadow the third
|
|
256
339
|
# check's positional one. Mirrors WorldTaskAssertionsGraderSchema.
|
|
@@ -262,72 +345,72 @@ def _parse_grader(grader: Any) -> dict | None:
|
|
|
262
345
|
fail(
|
|
263
346
|
"task",
|
|
264
347
|
f"check id '{key}' is used twice (an id of the form check_<n> is also the positional key of check n)",
|
|
265
|
-
f"/
|
|
348
|
+
f"{at}/checks/{i}/id",
|
|
266
349
|
)
|
|
267
350
|
fail(
|
|
268
351
|
"task",
|
|
269
352
|
f"check {i} has no id and its positional key '{key}' is taken by another check's id",
|
|
270
|
-
f"/
|
|
353
|
+
f"{at}/checks/{i}/id",
|
|
271
354
|
)
|
|
272
355
|
seen.add(key)
|
|
273
356
|
return {"kind": "assertions", "checks": parsed}
|
|
274
357
|
if kind == "rubric":
|
|
275
|
-
return _parse_rubric(grader)
|
|
276
|
-
fail("task", "grader.kind must be assertions, python or rubric", "/
|
|
358
|
+
return _parse_rubric(grader, at)
|
|
359
|
+
fail("task", "grader.kind must be assertions, python or rubric", f"{at}/kind")
|
|
277
360
|
raise AssertionError # fail() never returns
|
|
278
361
|
|
|
279
362
|
|
|
280
|
-
def _parse_rubric(grader: dict) -> dict:
|
|
363
|
+
def _parse_rubric(grader: dict, at: str) -> dict:
|
|
281
364
|
"""An LLM judge's brief. Checked here for shape only: the host stores it
|
|
282
365
|
with the task and never runs it; the platform grades it."""
|
|
283
366
|
if set(grader) - {"kind", "rubric", "criteria", "model"}:
|
|
284
|
-
fail("task", "a rubric grader is {kind, rubric, criteria?, model?}",
|
|
367
|
+
fail("task", "a rubric grader is {kind, rubric, criteria?, model?}", at)
|
|
285
368
|
rubric = grader.get("rubric")
|
|
286
369
|
if not isinstance(rubric, str) or not rubric:
|
|
287
|
-
fail("task", "a rubric grader needs its rubric text", "/
|
|
370
|
+
fail("task", "a rubric grader needs its rubric text", f"{at}/rubric")
|
|
288
371
|
if _bytes(rubric) > RUBRIC_BYTES:
|
|
289
|
-
fail("task", f"a rubric is at most {RUBRIC_BYTES} bytes", "/
|
|
372
|
+
fail("task", f"a rubric is at most {RUBRIC_BYTES} bytes", f"{at}/rubric")
|
|
290
373
|
out: dict = {"kind": "rubric", "rubric": rubric}
|
|
291
374
|
criteria = grader.get("criteria")
|
|
292
375
|
if criteria is not None:
|
|
293
376
|
if not isinstance(criteria, list) or not criteria:
|
|
294
|
-
fail("task", "criteria is a non-empty list", "/
|
|
377
|
+
fail("task", "criteria is a non-empty list", f"{at}/criteria")
|
|
295
378
|
if len(criteria) > RUBRIC_CRITERIA:
|
|
296
|
-
fail("task", f"a rubric has at most {RUBRIC_CRITERIA} criteria", "/
|
|
379
|
+
fail("task", f"a rubric has at most {RUBRIC_CRITERIA} criteria", f"{at}/criteria")
|
|
297
380
|
parsed = []
|
|
298
381
|
seen: set[str] = set()
|
|
299
382
|
for i, criterion in enumerate(criteria):
|
|
300
|
-
|
|
383
|
+
here = f"{at}/criteria/{i}"
|
|
301
384
|
if not isinstance(criterion, dict) or set(criterion) - {"id", "description", "weight"}:
|
|
302
|
-
fail("task", "a criterion is {id, description, weight?}",
|
|
385
|
+
fail("task", "a criterion is {id, description, weight?}", here)
|
|
303
386
|
criterion_id = criterion.get("id")
|
|
304
387
|
if not isinstance(criterion_id, str) or not _CRITERION_ID.match(criterion_id) or len(criterion_id) > 120:
|
|
305
|
-
fail("task", "a criterion id is letters, digits, _ . -", f"{
|
|
388
|
+
fail("task", "a criterion id is letters, digits, _ . -", f"{here}/id")
|
|
306
389
|
if criterion_id in seen:
|
|
307
|
-
fail("task", f"criterion id '{criterion_id}' is used twice", f"{
|
|
390
|
+
fail("task", f"criterion id '{criterion_id}' is used twice", f"{here}/id")
|
|
308
391
|
seen.add(criterion_id)
|
|
309
392
|
description = criterion.get("description")
|
|
310
393
|
if not isinstance(description, str) or not description:
|
|
311
|
-
fail("task", "a criterion needs a description", f"{
|
|
394
|
+
fail("task", "a criterion needs a description", f"{here}/description")
|
|
312
395
|
if _bytes(description) > RUBRIC_CRITERION_BYTES:
|
|
313
|
-
fail("task", f"a criterion is at most {RUBRIC_CRITERION_BYTES} bytes", f"{
|
|
396
|
+
fail("task", f"a criterion is at most {RUBRIC_CRITERION_BYTES} bytes", f"{here}/description")
|
|
314
397
|
item: dict = {"id": criterion_id, "description": description}
|
|
315
398
|
weight = criterion.get("weight")
|
|
316
399
|
if weight is not None:
|
|
317
400
|
if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0:
|
|
318
|
-
fail("task", "weight is a positive number", f"{
|
|
401
|
+
fail("task", "weight is a positive number", f"{here}/weight")
|
|
319
402
|
item["weight"] = float(weight)
|
|
320
403
|
parsed.append(item)
|
|
321
404
|
out["criteria"] = parsed
|
|
322
405
|
model = grader.get("model")
|
|
323
406
|
if model is not None:
|
|
324
407
|
if not isinstance(model, dict) or set(model) != {"provider", "model"}:
|
|
325
|
-
fail("task", "model is {provider, model}", "/
|
|
408
|
+
fail("task", "model is {provider, model}", f"{at}/model")
|
|
326
409
|
provider, name = model["provider"], model["model"]
|
|
327
410
|
if not isinstance(provider, str) or not 0 < len(provider) <= 120:
|
|
328
|
-
fail("task", "model.provider is a string of up to 120 characters", "/
|
|
411
|
+
fail("task", "model.provider is a string of up to 120 characters", f"{at}/model/provider")
|
|
329
412
|
if not isinstance(name, str) or not 0 < len(name) <= 200:
|
|
330
|
-
fail("task", "model.model is a string of up to 200 characters", "/
|
|
413
|
+
fail("task", "model.model is a string of up to 200 characters", f"{at}/model/model")
|
|
331
414
|
out["model"] = {"provider": provider, "model": name}
|
|
332
415
|
return out
|
|
333
416
|
|
|
@@ -416,6 +499,12 @@ def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
|
|
|
416
499
|
checks = root / "checks"
|
|
417
500
|
checks.mkdir(exist_ok=True)
|
|
418
501
|
(checks / "verify.py").write_text(spec.grader["source"], encoding="utf-8")
|
|
502
|
+
for verifier in spec.verifiers:
|
|
503
|
+
if verifier["kind"] != "python":
|
|
504
|
+
continue
|
|
505
|
+
source = verifier_source(root, verifier["name"])
|
|
506
|
+
source.parent.mkdir(parents=True, exist_ok=True)
|
|
507
|
+
source.write_text(verifier["source"], encoding="utf-8")
|
|
419
508
|
return root
|
|
420
509
|
|
|
421
510
|
|