@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
"""Who may do what to a world database, decided without naming an engine.
|
|
2
|
-
|
|
3
|
-
An engine reports every access a statement makes as an :class:`Access`; the
|
|
4
|
-
:class:`Guard` answers allow or deny from the door that is open. The doors are
|
|
5
|
-
the store's: a read-only query, a trusted verifier read, an ingest session, the
|
|
6
|
-
store's own bookkeeping, and the host's connection outside any of those.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
1
|
from __future__ import annotations
|
|
10
2
|
|
|
11
3
|
from abc import ABC, abstractmethod
|
|
@@ -49,18 +41,14 @@ READ_KINDS = frozenset({AccessKind.SELECT, AccessKind.READ, AccessKind.FUNCTION,
|
|
|
49
41
|
|
|
50
42
|
@dataclass(frozen=True)
|
|
51
43
|
class Access:
|
|
52
|
-
"""
|
|
53
|
-
|
|
54
|
-
ATTACH, transaction control), which every door but the trusted one refuses.
|
|
55
|
-
``target`` is the table or function name, ``schema`` the database it lives in
|
|
56
|
-
(``None`` when the engine does not say, :data:`STAGING` for the engine's
|
|
57
|
-
session-private staging space). ``unsafe`` marks a function the engine knows
|
|
58
|
-
reaches outside the database (the file system, a native extension)."""
|
|
44
|
+
"""Persistent DDL, PRAGMA, ATTACH and transaction control are ``OTHER``, which every door but the trusted one
|
|
45
|
+
refuses; ``schema`` is ``None`` when the engine does not say."""
|
|
59
46
|
|
|
60
47
|
kind: AccessKind
|
|
61
48
|
target: str | None = None
|
|
62
49
|
schema: str | None = None
|
|
63
50
|
unsafe: bool = False
|
|
51
|
+
column: str | None = None
|
|
64
52
|
|
|
65
53
|
@property
|
|
66
54
|
def unsafe_function(self) -> bool:
|
|
@@ -73,8 +61,7 @@ class Door(ABC):
|
|
|
73
61
|
|
|
74
62
|
|
|
75
63
|
class ReadOnlyDoor(Door):
|
|
76
|
-
"""``
|
|
77
|
-
``private`` the runtime context and events too (a verifier's trusted read)."""
|
|
64
|
+
"""``private`` is a verifier's trusted read: only it sees the runtime context and conversation events."""
|
|
78
65
|
|
|
79
66
|
def __init__(self, private: bool) -> None:
|
|
80
67
|
self.private = private
|
|
@@ -100,11 +87,9 @@ class TrustedDoor(Door):
|
|
|
100
87
|
|
|
101
88
|
|
|
102
89
|
class IngestDoor(Door):
|
|
103
|
-
"""
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
access is allowed whatever its name, so a TEMP object named like a main table shadows it (open
|
|
107
|
-
bug 2026-09-28-world-store-ingest-temp-shadow-bypasses-audit, to be fixed by #474)."""
|
|
90
|
+
"""Transaction control, PRAGMA, ATTACH, persistent DDL and the bookkeeping tables stay with the store, so a session
|
|
91
|
+
cannot commit early, disable enforcement or forge provenance. Known gap (#474): a TEMP object named like a main
|
|
92
|
+
table shadows it (bug 2026-09-28-world-store-ingest-temp-shadow-bypasses-audit)."""
|
|
108
93
|
|
|
109
94
|
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
110
95
|
if access.kind in READ_KINDS:
|
|
@@ -117,9 +102,8 @@ class IngestDoor(Door):
|
|
|
117
102
|
|
|
118
103
|
|
|
119
104
|
class HostDoor(Door):
|
|
120
|
-
"""The connection outside any store method
|
|
121
|
-
|
|
122
|
-
run behind the trusted door, so nothing of the store's passes through this one."""
|
|
105
|
+
"""The connection outside any store method; the store's own statements run behind the trusted door, so this one
|
|
106
|
+
stays narrow."""
|
|
123
107
|
|
|
124
108
|
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
125
109
|
if access.kind in READ_KINDS:
|
|
@@ -133,17 +117,18 @@ HOST = HostDoor()
|
|
|
133
117
|
|
|
134
118
|
|
|
135
119
|
class Guard:
|
|
136
|
-
"""
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
through an alias. Then the read-only door wins over the trusted one, which
|
|
141
|
-
wins over an ingest session, and the host door decides what is left."""
|
|
120
|
+
"""An attached store is decided first, whatever door is open: only a read of a table it exposes passes, so nothing
|
|
121
|
+
is ever written through an alias. Then read-only wins over trusted, trusted over ingest; the host door decides the
|
|
122
|
+
rest. A ``bare`` store's tables carry names no world may declare (the ``gateway_`` prefix), so a read naming
|
|
123
|
+
one without its store (``count(*)``) is that store's."""
|
|
142
124
|
|
|
143
125
|
def __init__(self, entities: frozenset[str], unsafe_functions: frozenset[str]) -> None:
|
|
144
126
|
self._entities = entities
|
|
145
127
|
self._unsafe = frozenset(name.lower() for name in unsafe_functions)
|
|
146
128
|
self._attached: dict[str, frozenset[str]] = {}
|
|
129
|
+
self._bare: dict[str, str] = {}
|
|
130
|
+
self._reads: dict[str, int] = {}
|
|
131
|
+
self._columns: dict[str, set[str]] = {}
|
|
147
132
|
self._reading: ReadOnlyDoor | None = None
|
|
148
133
|
self._trusted = 0
|
|
149
134
|
self._ingesting = False
|
|
@@ -156,20 +141,42 @@ class Guard:
|
|
|
156
141
|
def ingesting(self) -> bool:
|
|
157
142
|
return self._ingesting
|
|
158
143
|
|
|
159
|
-
def attach(self, name: str, tables: frozenset[str]) -> None:
|
|
144
|
+
def attach(self, name: str, tables: frozenset[str], *, bare: bool = False) -> None:
|
|
160
145
|
self._attached[name] = tables
|
|
146
|
+
self._reads[name] = 0
|
|
147
|
+
self._columns[name] = set()
|
|
148
|
+
if bare:
|
|
149
|
+
self._bare.update({table: name for table in tables})
|
|
150
|
+
|
|
151
|
+
def detach(self, name: str) -> None:
|
|
152
|
+
self._attached.pop(name, None)
|
|
153
|
+
self._reads.pop(name, None)
|
|
154
|
+
self._columns.pop(name, None)
|
|
155
|
+
self._bare = {table: owner for table, owner in self._bare.items() if owner != name}
|
|
161
156
|
|
|
162
157
|
def is_attached(self, name: str) -> bool:
|
|
163
158
|
return name in self._attached
|
|
164
159
|
|
|
160
|
+
def reads(self, name: str) -> int:
|
|
161
|
+
return self._reads.get(name, 0)
|
|
162
|
+
|
|
163
|
+
def columns(self, name: str) -> frozenset[str]:
|
|
164
|
+
return frozenset(self._columns.get(name, ()))
|
|
165
|
+
|
|
165
166
|
def allows(self, access: Access) -> bool:
|
|
166
167
|
access = self._marked(access)
|
|
167
|
-
if access.schema in self._attached:
|
|
168
|
-
return
|
|
169
|
-
|
|
168
|
+
if access.schema not in self._attached:
|
|
169
|
+
return self.door().allows(access, self._entities)
|
|
170
|
+
allowed = access.kind is AccessKind.READ and access.target in self._attached[access.schema]
|
|
171
|
+
self._reads[access.schema] += int(allowed)
|
|
172
|
+
if allowed and access.column:
|
|
173
|
+
self._columns[access.schema].add(access.column)
|
|
174
|
+
return allowed
|
|
170
175
|
|
|
171
176
|
def _marked(self, access: Access) -> Access:
|
|
172
|
-
"""
|
|
177
|
+
"""A bare name only an attached store holds reads that store; an unsafe function matches by name in any case."""
|
|
178
|
+
if access.kind is AccessKind.READ and access.schema is None and access.target in self._bare:
|
|
179
|
+
return replace(access, schema=self._bare[access.target])
|
|
173
180
|
if access.kind is not AccessKind.FUNCTION:
|
|
174
181
|
return access
|
|
175
182
|
return replace(access, unsafe=access.unsafe or (access.target or "").lower() in self._unsafe)
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""A filtered read of one entity, answered by the engine wherever it answers exactly.
|
|
2
|
+
|
|
3
|
+
:class:`Selection` splits a :class:`RowMatch` into the terms the engine compares
|
|
4
|
+
exactly as ``RowMatch`` does (equality on a top-level field typed string alone,
|
|
5
|
+
or integer alone with values that fit 64 bits, every value ASCII) and the rest.
|
|
6
|
+
The exact terms run in SQL over the field's column, counted and paged by the
|
|
7
|
+
engine; any other term is checked by ``RowMatch`` over the rows they left. Either
|
|
8
|
+
way the answer is ``RowMatch`` over every row, in key order.
|
|
9
|
+
|
|
10
|
+
:class:`ReadIndexes` are what make those terms a search: one per primary key,
|
|
11
|
+
unique key and relationship whose fields are all such fields, and one per field a
|
|
12
|
+
version's declared routes filter on, over each field's match text and then
|
|
13
|
+
``_key``, so an equality read finds its rows in key order and stops at its page.
|
|
14
|
+
They are not the contract's DDL: a store opens with or without them (each one
|
|
15
|
+
only exactly as its name derives it), a snapshot is written without them, and a
|
|
16
|
+
host builds them into the files its sessions read (the version's template, a
|
|
17
|
+
restored session).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
from typing import TYPE_CHECKING, Callable, Iterable
|
|
25
|
+
|
|
26
|
+
from ..compiler import quote, scalar_types
|
|
27
|
+
from .matching import RowMatch
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from .base import WorldDatabase
|
|
31
|
+
|
|
32
|
+
FORMAT = "1"
|
|
33
|
+
PREFIX = "_gateway_match"
|
|
34
|
+
INT64 = range(-(2**63), 2**63)
|
|
35
|
+
_INTEGER = re.compile(r"-?[0-9]+")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _int64_text(value: str) -> bool:
|
|
39
|
+
return value.isascii() and _INTEGER.fullmatch(value) is not None and int(value) in INT64
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
#: The field types whose stored text the engine's match text spells as Python's
|
|
43
|
+
#: ``str`` does, by type, with the values a term may compare exactly.
|
|
44
|
+
EXACT_VALUES: dict[frozenset[str], Callable[[str], bool]] = {
|
|
45
|
+
frozenset({"string"}): str.isascii,
|
|
46
|
+
frozenset({"integer"}): _int64_text,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _three(parts: list[str]) -> bool:
|
|
51
|
+
return len(parts) == 3
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _field_sets(entity: dict) -> list[list[str]]:
|
|
55
|
+
return [entity["primary_key"], *entity["unique"], *(rel["fields"] for rel in entity["relationships"])]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def exact_field(properties: dict, field: str) -> Callable[[str], bool] | None:
|
|
59
|
+
"""How a value sent for ``field`` is checked before the engine compares it, or
|
|
60
|
+
None when the engine cannot compare that field exactly (a dotted path, an
|
|
61
|
+
array, an object, a boolean, a number, a mix of types)."""
|
|
62
|
+
prop = properties.get(field) if "." not in field else None
|
|
63
|
+
if not isinstance(prop, dict):
|
|
64
|
+
return None
|
|
65
|
+
return EXACT_VALUES.get(frozenset(scalar_types(prop)))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
Fields = tuple[str, ...]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class ReadIndexes:
|
|
72
|
+
"""The read indexes of one store's contract, as the store's engine writes them:
|
|
73
|
+
its keys and relationships, and ``extra`` (entity, fields) a version's routes
|
|
74
|
+
filter on."""
|
|
75
|
+
|
|
76
|
+
def __init__(self, engine: type[WorldDatabase], entities: dict, extra: Iterable[tuple[str, Fields]] = ()) -> None:
|
|
77
|
+
self.engine = engine
|
|
78
|
+
self.entities = entities
|
|
79
|
+
self.extra = list(extra)
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def of(cls, store: WorldDatabase, extra: Iterable[tuple[str, Fields]] = ()) -> ReadIndexes:
|
|
83
|
+
return cls(type(store), store.contract.entities, extra)
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def named(names: Iterable[str]) -> list[str]:
|
|
87
|
+
"""The read index names among ``names``."""
|
|
88
|
+
return [name for name in names if name.split(":", 1)[0] == PREFIX]
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def parse(names: Iterable[str]) -> list[tuple[str, Fields]]:
|
|
92
|
+
"""The (entity, fields) each read index name says it covers; another name says none."""
|
|
93
|
+
parts = [name.split(":", 2) for name in ReadIndexes.named(names)]
|
|
94
|
+
return [(entity, tuple(fields.split(","))) for _, entity, fields in filter(_three, parts)]
|
|
95
|
+
|
|
96
|
+
def keys(self) -> list[tuple[str, Fields]]:
|
|
97
|
+
"""Every (entity, fields) a read index covers, each once, the fields all exact."""
|
|
98
|
+
declared = [(name, tuple(fields)) for name, entity in self.entities.items() for fields in _field_sets(entity)]
|
|
99
|
+
return list(dict.fromkeys(key for key in [*declared, *self.extra] if self._exact(*key)))
|
|
100
|
+
|
|
101
|
+
def _exact(self, entity: str, fields: Fields) -> bool:
|
|
102
|
+
declared = self.entities.get(entity)
|
|
103
|
+
properties = declared["schema"].get("properties", {}) if declared else {}
|
|
104
|
+
return bool(fields) and all(exact_field(properties, field) is not None for field in fields)
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def name(entity: str, fields: tuple[str, ...]) -> str:
|
|
108
|
+
return f"{PREFIX}:{entity}:{','.join(fields)}"
|
|
109
|
+
|
|
110
|
+
def by_name(self) -> dict[str, str]:
|
|
111
|
+
"""Each read index's name and the statement that builds it."""
|
|
112
|
+
return {
|
|
113
|
+
self.name(entity, fields): f"CREATE INDEX {quote(self.name(entity, fields))} ON {quote(entity)} "
|
|
114
|
+
f"({', '.join(self.engine.match_text(quote(field)) for field in fields)}, \"_key\")"
|
|
115
|
+
for entity, fields in self.keys()
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
def names(self) -> frozenset[str]:
|
|
119
|
+
return frozenset(self.by_name())
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Selection:
|
|
123
|
+
"""One read of ``entity`` through ``match``: :meth:`page` answers it."""
|
|
124
|
+
|
|
125
|
+
def __init__(self, store: WorldDatabase, entity: str, match: RowMatch) -> None:
|
|
126
|
+
self.store = store
|
|
127
|
+
self.entity = entity
|
|
128
|
+
self.match = match
|
|
129
|
+
properties = store.contract.entities[entity]["schema"].get("properties", {})
|
|
130
|
+
sent = [(arg, path) for arg, path in match.filters.items() if arg in match.values]
|
|
131
|
+
terms = [self._term(properties, arg, path) for arg, path in sent]
|
|
132
|
+
self.terms = [term for term in terms if term is not None]
|
|
133
|
+
self.exact = len(self.terms) == len(terms)
|
|
134
|
+
|
|
135
|
+
def _term(self, properties: dict, arg: str, path: str) -> tuple[str, list[str]] | None:
|
|
136
|
+
if self.match.modes.get(arg, "equals") != "equals":
|
|
137
|
+
return None
|
|
138
|
+
allowed = exact_field(properties, path)
|
|
139
|
+
sent = self.match.values[arg]
|
|
140
|
+
wanted = [str(value) for value in (sent if isinstance(sent, list) else [sent])]
|
|
141
|
+
if allowed is None or not all(allowed(value) for value in wanted):
|
|
142
|
+
return None
|
|
143
|
+
return path, [value.lower() for value in wanted]
|
|
144
|
+
|
|
145
|
+
def _where(self) -> tuple[str, list[str]]:
|
|
146
|
+
clauses = [
|
|
147
|
+
f"{self.store.match_text(quote(field))} IN ({', '.join('?' for _ in values)})"
|
|
148
|
+
for field, values in self.terms
|
|
149
|
+
]
|
|
150
|
+
params = [value for _, values in self.terms for value in values]
|
|
151
|
+
return (f" WHERE {' AND '.join(clauses)}" if clauses else ""), params
|
|
152
|
+
|
|
153
|
+
def statement(self, limit: int | None, offset: int) -> tuple[str, tuple]:
|
|
154
|
+
"""The engine's page of the exact terms, in key order."""
|
|
155
|
+
where, params = self._where()
|
|
156
|
+
clause, paging = self.store.page_clause(limit, offset)
|
|
157
|
+
return f"SELECT payload FROM {quote(self.entity)}{where} ORDER BY _key{clause}", (*params, *paging)
|
|
158
|
+
|
|
159
|
+
def page(self, limit: int | None, offset: int) -> tuple[list[dict], int]:
|
|
160
|
+
"""A page of the rows ``match`` selects and how many it selects in all."""
|
|
161
|
+
if any(not values for _, values in self.terms):
|
|
162
|
+
return [], 0
|
|
163
|
+
if self.exact:
|
|
164
|
+
where, params = self._where()
|
|
165
|
+
total = self.store.query(f"SELECT COUNT(*) FROM {quote(self.entity)}{where}", tuple(params))[0][0]
|
|
166
|
+
return [json.loads(row[0]) for row in self.store.query(*self.statement(limit, offset))], total
|
|
167
|
+
matched = self.match.select(json.loads(row[0]) for row in self.store.query(*self.statement(None, 0)))
|
|
168
|
+
end = None if limit is None else offset + limit
|
|
169
|
+
return matched[offset:end], len(matched)
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
"""The SQLite engine of the world database."""
|
|
2
|
-
|
|
3
1
|
from __future__ import annotations
|
|
4
2
|
|
|
5
3
|
import json
|
|
@@ -13,7 +11,7 @@ from typing import Any, Callable, Iterable
|
|
|
13
11
|
|
|
14
12
|
from ..compiler import json_pointer_parts, quote, scalar_types
|
|
15
13
|
from ..validation import ContractError, fail
|
|
16
|
-
from .base import STORED_LOCK_QUERY, WorldDatabase
|
|
14
|
+
from .base import STORED_LOCK_QUERY, SideTable, WorldDatabase
|
|
17
15
|
from .budget import QueryBudget
|
|
18
16
|
from .guard import STAGING, Access, AccessKind
|
|
19
17
|
|
|
@@ -41,19 +39,22 @@ MISSING_TABLE = re.compile(r"no such table: (.+?)\s*$", re.MULTILINE)
|
|
|
41
39
|
CARRIED_TABLE = "_gateway_calls"
|
|
42
40
|
SNAPSHOT_STEP_PAGES = 256
|
|
43
41
|
SNAPSHOT_STEP_PAUSE_S = 0.001
|
|
42
|
+
KELVIN_SIGN = 8490
|
|
44
43
|
|
|
45
44
|
|
|
46
45
|
def access(action: int, first: str | None, second: str | None, database: str | None) -> Access:
|
|
47
|
-
"""SQLite's authorizer
|
|
48
|
-
the function in ``second``; every other action names its object in ``first``."""
|
|
46
|
+
"""SQLite's authorizer names a called function in ``second`` and every other object in ``first``."""
|
|
49
47
|
kind = ACCESS_KINDS.get(action, AccessKind.OTHER)
|
|
50
48
|
target = second if kind is AccessKind.FUNCTION else first
|
|
51
|
-
|
|
49
|
+
column = second if kind is AccessKind.READ else None
|
|
50
|
+
return Access(kind, target, STAGING if database == TEMP_SCHEMA else database, column=column)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_SINGLE_QUOTED = re.compile(r"'(?:[^']|'')*'")
|
|
54
|
+
_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
52
55
|
|
|
53
56
|
|
|
54
57
|
def _without_comments(sql: str) -> str:
|
|
55
|
-
"""``sql`` with ``--`` line comments and ``/* */`` blocks removed, string
|
|
56
|
-
literals left alone (a ``--`` inside quotes is text, not a comment)."""
|
|
57
58
|
out: list[str] = []
|
|
58
59
|
i, n = 0, len(sql)
|
|
59
60
|
quoted: str | None = None
|
|
@@ -102,7 +103,7 @@ def _projection(field: str) -> str:
|
|
|
102
103
|
|
|
103
104
|
|
|
104
105
|
def _existing_file(path: Path) -> sqlite3.Connection:
|
|
105
|
-
"""
|
|
106
|
+
"""``mode=rw``: a missing file is refused, never created."""
|
|
106
107
|
return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=rw", uri=True)
|
|
107
108
|
|
|
108
109
|
|
|
@@ -118,10 +119,8 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
118
119
|
|
|
119
120
|
@classmethod
|
|
120
121
|
def ddl(cls, entities: dict) -> str:
|
|
121
|
-
"""The
|
|
122
|
-
|
|
123
|
-
verifier SQL can name it and SQLite enforces UNIQUE and foreign keys without
|
|
124
|
-
affinity coercion."""
|
|
122
|
+
"""The payload is the source of truth; fields are virtual columns so verifier SQL can name them and SQLite
|
|
123
|
+
enforces UNIQUE and foreign keys without affinity coercion."""
|
|
125
124
|
lines = [
|
|
126
125
|
"-- Generated by gateway-world-compiler/1; do not edit.",
|
|
127
126
|
"PRAGMA foreign_keys = ON;",
|
|
@@ -157,10 +156,7 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
157
156
|
|
|
158
157
|
@staticmethod
|
|
159
158
|
def split_statements(script: str) -> list[str]:
|
|
160
|
-
"""
|
|
161
|
-
semicolon inside a literal or a trigger body never splits. Text that is only
|
|
162
|
-
comments is not a statement and is dropped, so every reader of a verifier
|
|
163
|
-
(check, grade, the host) sees the same list."""
|
|
159
|
+
"""A ``;`` inside a literal or a trigger body never splits; pinned by ``sql-verifier-vectors.json``."""
|
|
164
160
|
statements: list[str] = []
|
|
165
161
|
buffer = ""
|
|
166
162
|
for line in script.splitlines(keepends=True):
|
|
@@ -200,9 +196,7 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
200
196
|
|
|
201
197
|
@staticmethod
|
|
202
198
|
def forbid_double_quoted_literals(conn: Any) -> None:
|
|
203
|
-
"""SQLite reads a double-quoted
|
|
204
|
-
verifier naming a missing column compares two literals and scores 0 without a word.
|
|
205
|
-
Switch the fallback off, or refuse to open where Python cannot (``setconfig`` is 3.12+)."""
|
|
199
|
+
"""SQLite reads a double-quoted missing column as a literal, so a verifier naming one would score 0 silently."""
|
|
206
200
|
if not hasattr(conn, "setconfig"):
|
|
207
201
|
fail(
|
|
208
202
|
"python_version",
|
|
@@ -223,13 +217,16 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
223
217
|
def _schema_objects(self) -> dict[str, str]:
|
|
224
218
|
return self._master(self._conn)
|
|
225
219
|
|
|
226
|
-
def
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
220
|
+
def _objects_of(self, script: str) -> dict[str, str]:
|
|
221
|
+
with closing(sqlite3.connect(":memory:")) as fresh:
|
|
222
|
+
fresh.executescript(script)
|
|
223
|
+
return self._master(fresh)
|
|
224
|
+
|
|
225
|
+
@classmethod
|
|
226
|
+
def match_text(cls, column: str) -> str:
|
|
227
|
+
# SQLite's lower() folds ASCII only; the one other character Python lowers to ASCII is the
|
|
228
|
+
# KELVIN SIGN ('K' -> 'k'), so it is folded first and an ASCII comparison equals RowMatch's.
|
|
229
|
+
return f"lower(replace(CAST({column} AS TEXT), char({KELVIN_SIGN}), 'k'))"
|
|
233
230
|
|
|
234
231
|
@staticmethod
|
|
235
232
|
def _master(conn: sqlite3.Connection) -> dict[str, str]:
|
|
@@ -264,7 +261,7 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
264
261
|
def _rollback_to(self, name: str) -> None:
|
|
265
262
|
self._conn.execute(f"ROLLBACK TO {name}")
|
|
266
263
|
|
|
267
|
-
def
|
|
264
|
+
def page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
|
|
268
265
|
return " LIMIT ? OFFSET ?", (-1 if limit is None else limit, offset)
|
|
269
266
|
|
|
270
267
|
def _relationships_broken(self) -> bool:
|
|
@@ -283,12 +280,29 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
283
280
|
def _attach_file(self, name: str, path: str | Path) -> None:
|
|
284
281
|
self._conn.execute("ATTACH DATABASE ? AS ?", (str(path), name))
|
|
285
282
|
|
|
283
|
+
def _detach_file(self, name: str) -> None:
|
|
284
|
+
self._conn.execute("DETACH DATABASE ?", (name,))
|
|
285
|
+
|
|
286
286
|
@staticmethod
|
|
287
287
|
def carry_entries(path: Path, entries: list[str]) -> None:
|
|
288
288
|
with closing(_existing_file(path)) as connection, connection:
|
|
289
289
|
connection.execute(f"CREATE TABLE {CARRIED_TABLE} (ordinal INTEGER PRIMARY KEY, entry TEXT NOT NULL)")
|
|
290
290
|
connection.executemany(f"INSERT INTO {CARRIED_TABLE} VALUES (?, ?)", list(enumerate(entries)))
|
|
291
291
|
|
|
292
|
+
@staticmethod
|
|
293
|
+
def identifiers(sql: str) -> frozenset[str]:
|
|
294
|
+
bare = _SINGLE_QUOTED.sub("''", _without_comments(sql))
|
|
295
|
+
return frozenset(name.lower() for name in _IDENTIFIER.findall(bare))
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def write_side_tables(path: Path, tables: Iterable[SideTable]) -> None:
|
|
299
|
+
with closing(sqlite3.connect(path)) as connection, connection:
|
|
300
|
+
for table in tables:
|
|
301
|
+
columns = ", ".join(f"{quote(column)} {kind}" for column, kind in table.columns)
|
|
302
|
+
connection.execute(f"CREATE TABLE {quote(table.name)} ({columns})")
|
|
303
|
+
marks = ", ".join("?" for _ in table.columns)
|
|
304
|
+
connection.executemany(f"INSERT INTO {quote(table.name)} VALUES ({marks})", table.rows)
|
|
305
|
+
|
|
292
306
|
@staticmethod
|
|
293
307
|
def take_entries(path: Path) -> list[str]:
|
|
294
308
|
with closing(_existing_file(path)) as connection, connection:
|
|
@@ -301,6 +315,19 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
301
315
|
connection.execute(f"DROP TABLE {CARRIED_TABLE}")
|
|
302
316
|
return entries
|
|
303
317
|
|
|
318
|
+
@staticmethod
|
|
319
|
+
def drop_indexes(path: Path, names: Iterable[str]) -> None:
|
|
320
|
+
with closing(_existing_file(path)) as connection, connection:
|
|
321
|
+
held = {name for (name,) in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")}
|
|
322
|
+
for name in sorted(set(names) & held):
|
|
323
|
+
connection.execute(f"DROP INDEX {quote(name)}")
|
|
324
|
+
|
|
325
|
+
@staticmethod
|
|
326
|
+
def recover_copy(source: Path, destination: Path) -> None:
|
|
327
|
+
# Opening the file read-write is what rolls back a hot journal its dead writer left beside it.
|
|
328
|
+
with closing(_existing_file(source)) as live, closing(sqlite3.connect(destination)) as copy:
|
|
329
|
+
live.backup(copy)
|
|
330
|
+
|
|
304
331
|
@staticmethod
|
|
305
332
|
def stored_lock(path: Path) -> dict:
|
|
306
333
|
connection = sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
|
|
@@ -332,9 +359,8 @@ class SQLiteWorldDatabase(WorldDatabase):
|
|
|
332
359
|
connection.close()
|
|
333
360
|
|
|
334
361
|
def _write_copy(self, destination: Path) -> None:
|
|
335
|
-
"""
|
|
336
|
-
|
|
337
|
-
to take it; the backup runs on this connection, so a write made then lands in the copy."""
|
|
362
|
+
"""Between steps the lock is let go for a waiting request; the backup runs on this connection, so a write made
|
|
363
|
+
then lands in the copy."""
|
|
338
364
|
target = sqlite3.connect(destination)
|
|
339
365
|
try:
|
|
340
366
|
with self._lock:
|