@withgateway/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +3971 -2338
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3950 -2318
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +214 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +206 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
- package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1456 -564
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1433 -557
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +211 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +203 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +96 -21
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +96 -21
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +300 -123
- package/dist/worlds/index.d.ts +300 -123
- package/dist/worlds/index.js +1385 -508
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1377 -501
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -515
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +22 -14
- package/skills/connector-schema-authoring/connector-worlds.md +26 -10
- package/skills/connector-schema-authoring/world-schema.md +111 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +16 -16
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""What one caller's SQL may cost a world database."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Iterable
|
|
9
|
+
|
|
10
|
+
BINARY = (bytes, bytearray, memoryview)
|
|
11
|
+
#: ``{"base64": ""}`` around an encoded blob, and the separator after each value.
|
|
12
|
+
BLOB_WRAPPER = len(json.dumps({"base64": ""}))
|
|
13
|
+
SEPARATOR = 2
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class QueryBudget:
|
|
18
|
+
"""Wall time for every statement in all, the largest single value the engine may make, and the
|
|
19
|
+
bytes the whole answer takes as JSON (what is sent, not what is counted in characters)."""
|
|
20
|
+
|
|
21
|
+
seconds: float
|
|
22
|
+
value_bytes: int
|
|
23
|
+
answer_bytes: int
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def size_of(row: Iterable[Any]) -> int:
|
|
27
|
+
return sum(QueryBudget._json_bytes(value) + SEPARATOR for value in row)
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def _json_bytes(value: Any) -> int:
|
|
31
|
+
if isinstance(value, BINARY):
|
|
32
|
+
return 4 * math.ceil(len(value) / 3) + BLOB_WRAPPER
|
|
33
|
+
return len(json.dumps(value))
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""The connections world-authored code holds: a tool's read-only ``conn`` and an
|
|
2
|
+
ingest session's raw SQL. Neither can commit, roll back or close; the database
|
|
3
|
+
owns every transaction."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable
|
|
8
|
+
|
|
9
|
+
from ..validation import ContractError, fail
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from .base import WorldDatabase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ReadCursor:
|
|
16
|
+
def __init__(self, rows: Iterable[Any]):
|
|
17
|
+
self._rows = iter(rows)
|
|
18
|
+
self.rowcount = -1
|
|
19
|
+
|
|
20
|
+
def fetchone(self):
|
|
21
|
+
return next(self._rows, None)
|
|
22
|
+
|
|
23
|
+
def fetchall(self):
|
|
24
|
+
return list(self._rows)
|
|
25
|
+
|
|
26
|
+
def __iter__(self):
|
|
27
|
+
return self._rows
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ReadConnection:
|
|
31
|
+
"""Compatibility conn= surface: execute SELECT only; no raw writes/commit."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, database: "WorldDatabase"):
|
|
34
|
+
self._database = database
|
|
35
|
+
|
|
36
|
+
def execute(self, sql: str, params: Any = ()) -> ReadCursor:
|
|
37
|
+
return ReadCursor(self._database.query(sql, tuple(params)))
|
|
38
|
+
|
|
39
|
+
def cursor(self):
|
|
40
|
+
fail(
|
|
41
|
+
"raw_write",
|
|
42
|
+
"Use conn.execute for reads and gateway_env.store for mutations",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def executescript(self, *args):
|
|
46
|
+
fail("raw_write", "Raw SQL scripts are unavailable in schema-backed tools")
|
|
47
|
+
|
|
48
|
+
def commit(self):
|
|
49
|
+
fail("raw_write", "The world owns the tool transaction")
|
|
50
|
+
|
|
51
|
+
rollback = commit
|
|
52
|
+
close = commit
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class IngestConnection:
|
|
56
|
+
"""Raw SQL inside one schema-gated ingest session.
|
|
57
|
+
|
|
58
|
+
Domain tables and TEMP objects are writable; the database owns the
|
|
59
|
+
transaction and commits only when every persisted row satisfies the contract
|
|
60
|
+
(known gap: a TEMP object named like a main table, see :class:`IngestSession`).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(self, database: "WorldDatabase"):
|
|
64
|
+
self._database = database
|
|
65
|
+
self._open = True
|
|
66
|
+
self.result: dict | None = None
|
|
67
|
+
|
|
68
|
+
def execute(self, sql: str, params: Any = ()) -> Any:
|
|
69
|
+
return self._run(lambda database: database._execute(sql, params))
|
|
70
|
+
|
|
71
|
+
def executemany(self, sql: str, rows: Any) -> Any:
|
|
72
|
+
return self._run(lambda database: database._executemany(sql, rows))
|
|
73
|
+
|
|
74
|
+
def script(self, sql: str) -> None:
|
|
75
|
+
"""Run a multi-statement script one statement at a time; an engine's own
|
|
76
|
+
script call may commit the session's transaction first."""
|
|
77
|
+
for statement in self._database.split_statements(sql):
|
|
78
|
+
self.execute(statement)
|
|
79
|
+
|
|
80
|
+
def _run(self, operation):
|
|
81
|
+
if not self._open:
|
|
82
|
+
fail("transaction", "The ingest session has ended")
|
|
83
|
+
database = self._database
|
|
84
|
+
database._key_error = None
|
|
85
|
+
try:
|
|
86
|
+
return operation(database)
|
|
87
|
+
except database.native_error as error:
|
|
88
|
+
context = {database.engine: str(error)}
|
|
89
|
+
if database._key_error is not None:
|
|
90
|
+
context["key"] = database._key_error
|
|
91
|
+
raise ContractError(
|
|
92
|
+
"ingest_sql",
|
|
93
|
+
"Statement rejected; only domain tables and TEMP objects are writable",
|
|
94
|
+
**context,
|
|
95
|
+
) from error
|
|
96
|
+
|
|
97
|
+
def commit(self):
|
|
98
|
+
fail("raw_write", "The store commits the ingest session when it ends")
|
|
99
|
+
|
|
100
|
+
rollback = commit
|
|
101
|
+
close = commit
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Who may do what to a world database, decided without naming an engine.
|
|
2
|
+
|
|
3
|
+
An engine reports every access a statement makes as an :class:`Access`; the
|
|
4
|
+
:class:`Guard` answers allow or deny from the door that is open. The doors are
|
|
5
|
+
the store's: a read-only query, a trusted verifier read, an ingest session, the
|
|
6
|
+
store's own bookkeeping, and the host's connection outside any of those.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from dataclasses import dataclass, replace
|
|
14
|
+
from enum import Enum
|
|
15
|
+
from typing import Iterator, Mapping
|
|
16
|
+
|
|
17
|
+
RUNTIME_TABLES = frozenset(
|
|
18
|
+
{
|
|
19
|
+
"benchmark_runtime_context",
|
|
20
|
+
"benchmark_runtime_parameters",
|
|
21
|
+
"benchmark_conversation_events",
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
RUNTIME_PARAMETERS = "benchmark_runtime_parameters"
|
|
25
|
+
PRIVATE_TABLE_ORDER = (
|
|
26
|
+
"_gateway_lineage",
|
|
27
|
+
"_gateway_batches",
|
|
28
|
+
"_gateway_contract",
|
|
29
|
+
"benchmark_runtime_context",
|
|
30
|
+
"benchmark_conversation_events",
|
|
31
|
+
)
|
|
32
|
+
PRIVATE_TABLES = frozenset(PRIVATE_TABLE_ORDER)
|
|
33
|
+
STAGING = "<staging>"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class AccessKind(Enum):
|
|
37
|
+
SELECT = "select"
|
|
38
|
+
READ = "read"
|
|
39
|
+
FUNCTION = "function"
|
|
40
|
+
RECURSIVE = "recursive"
|
|
41
|
+
WRITE = "write"
|
|
42
|
+
STAGE = "stage"
|
|
43
|
+
SAVEPOINT = "savepoint"
|
|
44
|
+
OTHER = "other"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
READ_KINDS = frozenset({AccessKind.SELECT, AccessKind.READ, AccessKind.FUNCTION, AccessKind.RECURSIVE})
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class Access:
|
|
52
|
+
"""One thing a statement does: read a table, call a function, write rows,
|
|
53
|
+
stage a TEMP object, open a savepoint, or anything else (DDL, PRAGMA,
|
|
54
|
+
ATTACH, transaction control), which every door but the trusted one refuses.
|
|
55
|
+
``target`` is the table or function name, ``schema`` the database it lives in
|
|
56
|
+
(``None`` when the engine does not say, :data:`STAGING` for the engine's
|
|
57
|
+
session-private staging space). ``unsafe`` marks a function the engine knows
|
|
58
|
+
reaches outside the database (the file system, a native extension)."""
|
|
59
|
+
|
|
60
|
+
kind: AccessKind
|
|
61
|
+
target: str | None = None
|
|
62
|
+
schema: str | None = None
|
|
63
|
+
unsafe: bool = False
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def unsafe_function(self) -> bool:
|
|
67
|
+
return self.kind is AccessKind.FUNCTION and self.unsafe
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Door(ABC):
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool: ...
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ReadOnlyDoor(Door):
|
|
76
|
+
"""``query``: reads of domain tables and the runtime parameters; with
|
|
77
|
+
``private`` the runtime context and events too (a verifier's trusted read)."""
|
|
78
|
+
|
|
79
|
+
def __init__(self, private: bool) -> None:
|
|
80
|
+
self.private = private
|
|
81
|
+
|
|
82
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
83
|
+
if access.kind not in READ_KINDS:
|
|
84
|
+
return False
|
|
85
|
+
if access.kind is AccessKind.READ and not self._readable(access.target, entities):
|
|
86
|
+
return False
|
|
87
|
+
return not access.unsafe_function
|
|
88
|
+
|
|
89
|
+
def _readable(self, table: str | None, entities: frozenset[str]) -> bool:
|
|
90
|
+
if table in entities or table == RUNTIME_PARAMETERS:
|
|
91
|
+
return True
|
|
92
|
+
return self.private and table in RUNTIME_TABLES
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class TrustedDoor(Door):
|
|
96
|
+
"""The store's own statements: bookkeeping, audits, transaction control."""
|
|
97
|
+
|
|
98
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
99
|
+
return True
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class IngestDoor(Door):
|
|
103
|
+
"""Domain tables and TEMP staging are writable; transaction control, PRAGMA,
|
|
104
|
+
ATTACH, persistent DDL and the private bookkeeping tables stay with the store,
|
|
105
|
+
so a session cannot commit early, disable enforcement, or forge provenance. Known gap: a STAGE
|
|
106
|
+
access is allowed whatever its name, so a TEMP object named like a main table shadows it (open
|
|
107
|
+
bug 2026-09-28-world-store-ingest-temp-shadow-bypasses-audit, to be fixed by #474)."""
|
|
108
|
+
|
|
109
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
110
|
+
if access.kind in READ_KINDS:
|
|
111
|
+
return (
|
|
112
|
+
not (access.kind is AccessKind.READ and access.target in PRIVATE_TABLES) and not access.unsafe_function
|
|
113
|
+
)
|
|
114
|
+
if access.kind is AccessKind.WRITE:
|
|
115
|
+
return access.target in entities or access.schema == STAGING
|
|
116
|
+
return access.kind in (AccessKind.STAGE, AccessKind.SAVEPOINT)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class HostDoor(Door):
|
|
120
|
+
"""The connection outside any store method: it reads its own file, writes only
|
|
121
|
+
the runtime tables, and calls no unsafe function. The store's own statements all
|
|
122
|
+
run behind the trusted door, so nothing of the store's passes through this one."""
|
|
123
|
+
|
|
124
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
125
|
+
if access.kind in READ_KINDS:
|
|
126
|
+
return not access.unsafe_function
|
|
127
|
+
return access.kind is AccessKind.WRITE and access.target in RUNTIME_TABLES
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
TRUSTED = TrustedDoor()
|
|
131
|
+
INGEST = IngestDoor()
|
|
132
|
+
HOST = HostDoor()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class Guard:
|
|
136
|
+
"""The open door of one database connection and the attached stores it reads.
|
|
137
|
+
|
|
138
|
+
An access to an attached store is decided first, whatever door is open: only
|
|
139
|
+
a read of a table that store exposes passes, so nothing is ever written
|
|
140
|
+
through an alias. Then the read-only door wins over the trusted one, which
|
|
141
|
+
wins over an ingest session, and the host door decides what is left."""
|
|
142
|
+
|
|
143
|
+
def __init__(self, entities: frozenset[str], unsafe_functions: frozenset[str]) -> None:
|
|
144
|
+
self._entities = entities
|
|
145
|
+
self._unsafe = frozenset(name.lower() for name in unsafe_functions)
|
|
146
|
+
self._attached: dict[str, frozenset[str]] = {}
|
|
147
|
+
self._reading: ReadOnlyDoor | None = None
|
|
148
|
+
self._trusted = 0
|
|
149
|
+
self._ingesting = False
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def attached(self) -> Mapping[str, frozenset[str]]:
|
|
153
|
+
return dict(self._attached)
|
|
154
|
+
|
|
155
|
+
@property
|
|
156
|
+
def ingesting(self) -> bool:
|
|
157
|
+
return self._ingesting
|
|
158
|
+
|
|
159
|
+
def attach(self, name: str, tables: frozenset[str]) -> None:
|
|
160
|
+
self._attached[name] = tables
|
|
161
|
+
|
|
162
|
+
def is_attached(self, name: str) -> bool:
|
|
163
|
+
return name in self._attached
|
|
164
|
+
|
|
165
|
+
def allows(self, access: Access) -> bool:
|
|
166
|
+
access = self._marked(access)
|
|
167
|
+
if access.schema in self._attached:
|
|
168
|
+
return access.kind is AccessKind.READ and access.target in self._attached[access.schema]
|
|
169
|
+
return self.door().allows(access, self._entities)
|
|
170
|
+
|
|
171
|
+
def _marked(self, access: Access) -> Access:
|
|
172
|
+
"""``access`` with ``unsafe`` set when it calls a function the engine declared unsafe, by name in any case."""
|
|
173
|
+
if access.kind is not AccessKind.FUNCTION:
|
|
174
|
+
return access
|
|
175
|
+
return replace(access, unsafe=access.unsafe or (access.target or "").lower() in self._unsafe)
|
|
176
|
+
|
|
177
|
+
def door(self) -> Door:
|
|
178
|
+
if self._reading is not None:
|
|
179
|
+
return self._reading
|
|
180
|
+
if self._trusted:
|
|
181
|
+
return TRUSTED
|
|
182
|
+
if self._ingesting:
|
|
183
|
+
return INGEST
|
|
184
|
+
return HOST
|
|
185
|
+
|
|
186
|
+
@contextmanager
|
|
187
|
+
def reading(self, private: bool) -> Iterator[None]:
|
|
188
|
+
self._reading = ReadOnlyDoor(private)
|
|
189
|
+
try:
|
|
190
|
+
yield
|
|
191
|
+
finally:
|
|
192
|
+
self._reading = None
|
|
193
|
+
|
|
194
|
+
@contextmanager
|
|
195
|
+
def trusted(self) -> Iterator[None]:
|
|
196
|
+
self._trusted += 1
|
|
197
|
+
try:
|
|
198
|
+
yield
|
|
199
|
+
finally:
|
|
200
|
+
self._trusted -= 1
|
|
201
|
+
|
|
202
|
+
@contextmanager
|
|
203
|
+
def ingest(self) -> Iterator[None]:
|
|
204
|
+
self._ingesting = True
|
|
205
|
+
try:
|
|
206
|
+
yield
|
|
207
|
+
finally:
|
|
208
|
+
self._ingesting = False
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""One schema-gated raw SQL session: write freely, commit only if every row conforms
|
|
2
|
+
(known gap: a TEMP object named like a main table, see :class:`IngestSession`)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import uuid
|
|
7
|
+
from typing import TYPE_CHECKING, Iterator
|
|
8
|
+
|
|
9
|
+
from ..compiler import quote
|
|
10
|
+
from ..validation import canonical, digest, encoded, fail, members, nonempty
|
|
11
|
+
from .connections import IngestConnection
|
|
12
|
+
from .provenance import validate_source
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from .base import WorldDatabase
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class IngestSession:
|
|
19
|
+
"""The session :meth:`WorldDatabase.ingest` opens. The database's guard is on its
|
|
20
|
+
ingest door while the caller holds the connection; on exit every persisted row is
|
|
21
|
+
audited, and the changes are recorded as one batch with a lineage row per touched
|
|
22
|
+
key, or the whole session rolls back. Known gap: a TEMP table named like a domain or
|
|
23
|
+
bookkeeping table shadows it for the audit and the provenance writes (bug log
|
|
24
|
+
``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, pinned xfail in the guard matrix,
|
|
25
|
+
to be fixed by #474)."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, database: "WorldDatabase", *, source: dict, mapping: dict, batch_id: str | None, limit: int):
|
|
28
|
+
validate_source(source, "/ingest/source")
|
|
29
|
+
members(mapping, {"id", "version"}, {"id", "version"}, "/ingest/mapping")
|
|
30
|
+
for key, value in mapping.items():
|
|
31
|
+
nonempty(value, f"/ingest/mapping/{key}")
|
|
32
|
+
self._database = database
|
|
33
|
+
self._source = source
|
|
34
|
+
self._mapping = mapping
|
|
35
|
+
self._batch_id = batch_id
|
|
36
|
+
self._limit = limit
|
|
37
|
+
|
|
38
|
+
def run(self) -> Iterator[IngestConnection]:
|
|
39
|
+
database = self._database
|
|
40
|
+
with database._lock:
|
|
41
|
+
if database._depth or database._guard.ingesting:
|
|
42
|
+
fail(
|
|
43
|
+
"transaction",
|
|
44
|
+
"Ingest needs no active tool, batch or ingest transaction",
|
|
45
|
+
)
|
|
46
|
+
batch = self._batch_id if self._batch_id is not None else uuid.uuid4().hex
|
|
47
|
+
nonempty(batch, "/ingest/id")
|
|
48
|
+
with database._trusted():
|
|
49
|
+
applied = database._execute("SELECT 1 FROM _gateway_batches WHERE id = ?", (batch,)).fetchone()
|
|
50
|
+
if applied:
|
|
51
|
+
fail(
|
|
52
|
+
"idempotency_conflict",
|
|
53
|
+
"Ingest ID was already applied; raw sessions are not replayable",
|
|
54
|
+
"/ingest/id",
|
|
55
|
+
batch=batch,
|
|
56
|
+
)
|
|
57
|
+
before = self._state()
|
|
58
|
+
connection = IngestConnection(database)
|
|
59
|
+
with database._trusted():
|
|
60
|
+
database._begin_write()
|
|
61
|
+
database._depth = 1
|
|
62
|
+
try:
|
|
63
|
+
with database._guard.ingest():
|
|
64
|
+
changes = yield from self._session(connection, before, batch)
|
|
65
|
+
finally:
|
|
66
|
+
database._depth = 0
|
|
67
|
+
connection._open = False
|
|
68
|
+
with database._trusted():
|
|
69
|
+
database._drop_staging()
|
|
70
|
+
connection.result = self._result(batch, changes)
|
|
71
|
+
|
|
72
|
+
def _session(self, connection: IngestConnection, before: dict, batch: str):
|
|
73
|
+
database = self._database
|
|
74
|
+
try:
|
|
75
|
+
yield connection
|
|
76
|
+
violations = database._audit.violations(self._limit)
|
|
77
|
+
if violations:
|
|
78
|
+
fail(
|
|
79
|
+
"ingest_rejected",
|
|
80
|
+
"Persisted rows violate the contract; nothing was committed",
|
|
81
|
+
batch=batch,
|
|
82
|
+
violations=[error.detail for error in violations],
|
|
83
|
+
truncated=len(violations) >= self._limit,
|
|
84
|
+
)
|
|
85
|
+
changes = self._changes(before, self._state())
|
|
86
|
+
with database._trusted():
|
|
87
|
+
self._record(batch, changes)
|
|
88
|
+
database._commit()
|
|
89
|
+
return changes
|
|
90
|
+
except BaseException:
|
|
91
|
+
with database._trusted():
|
|
92
|
+
database._rollback()
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
def _record(self, batch: str, changes: list) -> None:
|
|
96
|
+
if not changes:
|
|
97
|
+
return
|
|
98
|
+
database = self._database
|
|
99
|
+
metadata = {
|
|
100
|
+
"id": batch,
|
|
101
|
+
"kind": "ingest",
|
|
102
|
+
"target": database.contract.target,
|
|
103
|
+
"mapping": self._mapping,
|
|
104
|
+
"coverage": "partial",
|
|
105
|
+
}
|
|
106
|
+
database._execute(
|
|
107
|
+
"INSERT INTO _gateway_batches VALUES (?, ?, ?)",
|
|
108
|
+
(batch, digest(encoded(changes)), canonical(metadata)),
|
|
109
|
+
)
|
|
110
|
+
database._executemany(
|
|
111
|
+
"INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
|
|
112
|
+
[(batch, index, entity, key, canonical(self._source)) for index, (entity, key, _) in enumerate(changes)],
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def _state(self) -> dict[str, dict[str, str]]:
|
|
116
|
+
"""Persisted ``_key -> payload`` per entity, for change attribution."""
|
|
117
|
+
database = self._database
|
|
118
|
+
with database._trusted():
|
|
119
|
+
return {
|
|
120
|
+
name: dict(database._execute(f"SELECT _key, payload FROM {quote(name)}"))
|
|
121
|
+
for name in database.contract.entities
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def _changes(before: dict, after: dict) -> list[list[str]]:
|
|
126
|
+
changes = []
|
|
127
|
+
for name in sorted(before):
|
|
128
|
+
old, new = before[name], after[name]
|
|
129
|
+
for key in sorted(set(old) | set(new)):
|
|
130
|
+
if old.get(key) == new.get(key):
|
|
131
|
+
continue
|
|
132
|
+
changes.append([name, key, IngestSession._change(key, old, new)])
|
|
133
|
+
return changes
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def _change(key: str, old: dict, new: dict) -> str:
|
|
137
|
+
if key not in old:
|
|
138
|
+
return "insert"
|
|
139
|
+
if key not in new:
|
|
140
|
+
return "delete"
|
|
141
|
+
return "update"
|
|
142
|
+
|
|
143
|
+
@staticmethod
|
|
144
|
+
def _result(batch: str, changes: list) -> dict:
|
|
145
|
+
counts = {op: 0 for op in ("insert", "update", "delete")}
|
|
146
|
+
for _, _, op in changes:
|
|
147
|
+
counts[op] += 1
|
|
148
|
+
return {
|
|
149
|
+
"status": "applied" if changes else "unchanged",
|
|
150
|
+
"batch": batch,
|
|
151
|
+
"inserted": counts["insert"],
|
|
152
|
+
"updated": counts["update"],
|
|
153
|
+
"deleted": counts["delete"],
|
|
154
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""The one rule by which a world's rows match a filter: the store's ``list_where``,
|
|
2
|
+
an api world's routes and ``worlds data query`` all answer with it."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any, Callable, Iterable
|
|
8
|
+
|
|
9
|
+
COMPARISONS: dict[str, Callable[[str, str], bool]] = {
|
|
10
|
+
"equals": lambda have, wanted: have == wanted,
|
|
11
|
+
"contains": lambda have, wanted: wanted in have,
|
|
12
|
+
"prefix": lambda have, wanted: have.startswith(wanted),
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def dig(row: Any, path: str) -> Any:
|
|
17
|
+
for part in path.split("."):
|
|
18
|
+
if not isinstance(row, dict) or part not in row:
|
|
19
|
+
return None
|
|
20
|
+
row = row[part]
|
|
21
|
+
return row
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class RowMatch:
|
|
26
|
+
"""Every filter whose argument was supplied must match; a parameter left out does not narrow.
|
|
27
|
+
|
|
28
|
+
``filters`` maps an argument to the dotted field path it reads, ``values`` the
|
|
29
|
+
argument to the value (or values) sent. ``modes`` picks how: ``equals``
|
|
30
|
+
(default), ``contains`` (case-insensitive substring, the vendor's ``icontains``)
|
|
31
|
+
or ``prefix``. The stored value is compared as text, case-insensitively; a list
|
|
32
|
+
field matches when any element does, and a parameter sent more than once
|
|
33
|
+
(``status[]=a&status[]=b``) matches when any of its values does. A missing field
|
|
34
|
+
never matches."""
|
|
35
|
+
|
|
36
|
+
filters: dict[str, str]
|
|
37
|
+
values: dict[str, Any]
|
|
38
|
+
modes: dict[str, str] = field(default_factory=dict)
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def on_fields(cls, values: dict[str, Any]) -> "RowMatch":
|
|
42
|
+
return cls({name: name for name in values}, values)
|
|
43
|
+
|
|
44
|
+
def matches(self, row: dict) -> bool:
|
|
45
|
+
return all(self._field_matches(row, arg, path) for arg, path in self.filters.items() if arg in self.values)
|
|
46
|
+
|
|
47
|
+
def select(self, rows: Iterable[dict]) -> list[dict]:
|
|
48
|
+
return [row for row in rows if self.matches(row)]
|
|
49
|
+
|
|
50
|
+
def _field_matches(self, row: dict, arg: str, path: str) -> bool:
|
|
51
|
+
value = dig(row, path)
|
|
52
|
+
if value is None:
|
|
53
|
+
return False
|
|
54
|
+
compare = COMPARISONS.get(self.modes.get(arg, "equals"), COMPARISONS["equals"])
|
|
55
|
+
sent = self.values[arg]
|
|
56
|
+
wanted = [w.lower() for w in (sent if isinstance(sent, list) else [sent])]
|
|
57
|
+
candidates = value if isinstance(value, list) else [value]
|
|
58
|
+
return any(compare(str(candidate).lower(), w) for candidate in candidates for w in wanted)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""The provenance a mutation carries: the record it came from and the evidence behind it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ..validation import fail, members, nonempty
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_source(source: Any, location: str) -> None:
|
|
11
|
+
members(source, {"record", "evidence"}, {"record"}, location)
|
|
12
|
+
nonempty(source["record"], f"{location}/record")
|
|
13
|
+
evidence = source.get("evidence", [])
|
|
14
|
+
if not isinstance(evidence, list):
|
|
15
|
+
fail(
|
|
16
|
+
"provenance",
|
|
17
|
+
"evidence must be an array of stable reference/digest pairs",
|
|
18
|
+
location,
|
|
19
|
+
)
|
|
20
|
+
for index, entry in enumerate(evidence):
|
|
21
|
+
members(
|
|
22
|
+
entry,
|
|
23
|
+
{"reference", "digest"},
|
|
24
|
+
{"reference", "digest"},
|
|
25
|
+
f"{location}/evidence/{index}",
|
|
26
|
+
)
|
|
27
|
+
nonempty(entry["reference"], location)
|
|
28
|
+
value = entry["digest"]
|
|
29
|
+
if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
|
|
30
|
+
fail("provenance", "Evidence digest must be lowercase SHA-256 hex", location)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""The row a world database answers a query with, whatever engine read it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Iterable, Sequence
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Row(tuple):
|
|
9
|
+
"""One result row: a tuple of its values that also answers by column name, so
|
|
10
|
+
``row[0]``, ``row["name"]``, ``dict(row)`` and ``tuple(row) == (...)`` hold on
|
|
11
|
+
every engine. A name is matched exactly, then case-insensitively; with two
|
|
12
|
+
columns of one name, the first answers."""
|
|
13
|
+
|
|
14
|
+
def __new__(cls, columns: Sequence[str], values: Iterable[Any]) -> "Row":
|
|
15
|
+
row = super().__new__(cls, values)
|
|
16
|
+
row._columns = tuple(columns)
|
|
17
|
+
return row
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def all(cls, description: Sequence[Sequence[Any]] | None, values: Iterable[Iterable[Any]]) -> "Rows":
|
|
21
|
+
"""Every row of a result, named by a DB-API ``cursor.description``."""
|
|
22
|
+
columns = tuple(column[0] for column in description or ())
|
|
23
|
+
return Rows(columns, (cls(columns, row) for row in values))
|
|
24
|
+
|
|
25
|
+
def keys(self) -> list[str]:
|
|
26
|
+
return list(self._columns)
|
|
27
|
+
|
|
28
|
+
def __getitem__(self, key: Any) -> Any:
|
|
29
|
+
if not isinstance(key, str):
|
|
30
|
+
return super().__getitem__(key)
|
|
31
|
+
return super().__getitem__(self._index(key))
|
|
32
|
+
|
|
33
|
+
def _index(self, name: str) -> int:
|
|
34
|
+
if name in self._columns:
|
|
35
|
+
return self._columns.index(name)
|
|
36
|
+
folded = [column.lower() for column in self._columns]
|
|
37
|
+
if name.lower() in folded:
|
|
38
|
+
return folded.index(name.lower())
|
|
39
|
+
raise IndexError(f"No column named {name}")
|
|
40
|
+
|
|
41
|
+
def __getnewargs__(self) -> tuple:
|
|
42
|
+
return (self._columns, tuple(self))
|
|
43
|
+
|
|
44
|
+
def __repr__(self) -> str:
|
|
45
|
+
return f"Row({dict(zip(self._columns, self))!r})"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Rows(list):
|
|
49
|
+
"""A query's rows, which also name the columns when there are none."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, columns: Sequence[str], rows: Iterable[Row]) -> None:
|
|
52
|
+
super().__init__(rows)
|
|
53
|
+
self.columns = tuple(columns)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""The statements that write the runtime tables a verifier reads: the active task,
|
|
2
|
+
the run's condition bindings and the turn events. A world database and a
|
|
3
|
+
mock-tools world's raw file run the same statements."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
Statement = tuple[str, tuple]
|
|
8
|
+
|
|
9
|
+
TASK_CONTEXT = "benchmark_runtime_context"
|
|
10
|
+
RUN_PARAMETERS = "benchmark_runtime_parameters"
|
|
11
|
+
CONVERSATION_EVENTS = "benchmark_conversation_events"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RuntimeRows:
|
|
15
|
+
@staticmethod
|
|
16
|
+
def task_context(scenario_id: str) -> list[Statement]:
|
|
17
|
+
return [
|
|
18
|
+
(f"DELETE FROM {TASK_CONTEXT}", ()),
|
|
19
|
+
(f"INSERT INTO {TASK_CONTEXT} (context_key, scenario_id) VALUES ('active_task', ?)", (scenario_id,)),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def run_parameters(parameters: dict[str, str]) -> list[Statement]:
|
|
24
|
+
return [(f"DELETE FROM {RUN_PARAMETERS}", ())] + [
|
|
25
|
+
(f"INSERT INTO {RUN_PARAMETERS} (key, value) VALUES (?, ?)", (key, parameters[key]))
|
|
26
|
+
for key in sorted(parameters)
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def conversation_event(
|
|
31
|
+
scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
|
|
32
|
+
) -> list[Statement]:
|
|
33
|
+
return [
|
|
34
|
+
(
|
|
35
|
+
f"INSERT INTO {CONVERSATION_EVENTS} "
|
|
36
|
+
"(scenario_id, user_turn_index, event_type, actor, name, payload_json) VALUES (?, ?, ?, ?, ?, ?)",
|
|
37
|
+
(scenario_id, user_turn_index, event_type, actor, name, payload_json),
|
|
38
|
+
)
|
|
39
|
+
]
|