@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""A filtered read of one entity, answered by the engine wherever it answers exactly.
|
|
2
|
+
|
|
3
|
+
:class:`Selection` splits a :class:`RowMatch` into the terms the engine compares
|
|
4
|
+
exactly as ``RowMatch`` does (equality on a top-level field typed string alone,
|
|
5
|
+
or integer alone with values that fit 64 bits, every value ASCII) and the rest.
|
|
6
|
+
The exact terms run in SQL over the field's column, counted and paged by the
|
|
7
|
+
engine; any other term is checked by ``RowMatch`` over the rows they left. Either
|
|
8
|
+
way the answer is ``RowMatch`` over every row, in key order.
|
|
9
|
+
|
|
10
|
+
:class:`ReadIndexes` are what make those terms a search: one per primary key,
|
|
11
|
+
unique key and relationship whose fields are all such fields, and one per field a
|
|
12
|
+
version's declared routes filter on, over each field's match text and then
|
|
13
|
+
``_key``, so an equality read finds its rows in key order and stops at its page.
|
|
14
|
+
They are not the contract's DDL: a store opens with or without them (each one
|
|
15
|
+
only exactly as its name derives it), a snapshot is written without them, and a
|
|
16
|
+
host builds them into the files its sessions read (the version's template, a
|
|
17
|
+
restored session).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
from typing import TYPE_CHECKING, Callable, Iterable
|
|
25
|
+
|
|
26
|
+
from ..compiler import quote, scalar_types
|
|
27
|
+
from .matching import RowMatch
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from .base import WorldDatabase
|
|
31
|
+
|
|
32
|
+
FORMAT = "1"
|
|
33
|
+
PREFIX = "_gateway_match"
|
|
34
|
+
INT64 = range(-(2**63), 2**63)
|
|
35
|
+
_INTEGER = re.compile(r"-?[0-9]+")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _int64_text(value: str) -> bool:
|
|
39
|
+
return value.isascii() and _INTEGER.fullmatch(value) is not None and int(value) in INT64
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
#: The field types whose stored text the engine's match text spells as Python's
|
|
43
|
+
#: ``str`` does, by type, with the values a term may compare exactly.
|
|
44
|
+
EXACT_VALUES: dict[frozenset[str], Callable[[str], bool]] = {
|
|
45
|
+
frozenset({"string"}): str.isascii,
|
|
46
|
+
frozenset({"integer"}): _int64_text,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _three(parts: list[str]) -> bool:
|
|
51
|
+
return len(parts) == 3
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _field_sets(entity: dict) -> list[list[str]]:
|
|
55
|
+
return [entity["primary_key"], *entity["unique"], *(rel["fields"] for rel in entity["relationships"])]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def exact_field(properties: dict, field: str) -> Callable[[str], bool] | None:
|
|
59
|
+
"""How a value sent for ``field`` is checked before the engine compares it, or
|
|
60
|
+
None when the engine cannot compare that field exactly (a dotted path, an
|
|
61
|
+
array, an object, a boolean, a number, a mix of types)."""
|
|
62
|
+
prop = properties.get(field) if "." not in field else None
|
|
63
|
+
if not isinstance(prop, dict):
|
|
64
|
+
return None
|
|
65
|
+
return EXACT_VALUES.get(frozenset(scalar_types(prop)))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
Fields = tuple[str, ...]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class ReadIndexes:
|
|
72
|
+
"""The read indexes of one store's contract, as the store's engine writes them:
|
|
73
|
+
its keys and relationships, and ``extra`` (entity, fields) a version's routes
|
|
74
|
+
filter on."""
|
|
75
|
+
|
|
76
|
+
def __init__(self, engine: type[WorldDatabase], entities: dict, extra: Iterable[tuple[str, Fields]] = ()) -> None:
|
|
77
|
+
self.engine = engine
|
|
78
|
+
self.entities = entities
|
|
79
|
+
self.extra = list(extra)
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def of(cls, store: WorldDatabase, extra: Iterable[tuple[str, Fields]] = ()) -> ReadIndexes:
|
|
83
|
+
return cls(type(store), store.contract.entities, extra)
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def named(names: Iterable[str]) -> list[str]:
|
|
87
|
+
"""The read index names among ``names``."""
|
|
88
|
+
return [name for name in names if name.split(":", 1)[0] == PREFIX]
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def parse(names: Iterable[str]) -> list[tuple[str, Fields]]:
|
|
92
|
+
"""The (entity, fields) each read index name says it covers; another name says none."""
|
|
93
|
+
parts = [name.split(":", 2) for name in ReadIndexes.named(names)]
|
|
94
|
+
return [(entity, tuple(fields.split(","))) for _, entity, fields in filter(_three, parts)]
|
|
95
|
+
|
|
96
|
+
def keys(self) -> list[tuple[str, Fields]]:
|
|
97
|
+
"""Every (entity, fields) a read index covers, each once, the fields all exact."""
|
|
98
|
+
declared = [(name, tuple(fields)) for name, entity in self.entities.items() for fields in _field_sets(entity)]
|
|
99
|
+
return list(dict.fromkeys(key for key in [*declared, *self.extra] if self._exact(*key)))
|
|
100
|
+
|
|
101
|
+
def _exact(self, entity: str, fields: Fields) -> bool:
|
|
102
|
+
declared = self.entities.get(entity)
|
|
103
|
+
properties = declared["schema"].get("properties", {}) if declared else {}
|
|
104
|
+
return bool(fields) and all(exact_field(properties, field) is not None for field in fields)
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def name(entity: str, fields: tuple[str, ...]) -> str:
|
|
108
|
+
return f"{PREFIX}:{entity}:{','.join(fields)}"
|
|
109
|
+
|
|
110
|
+
def by_name(self) -> dict[str, str]:
|
|
111
|
+
"""Each read index's name and the statement that builds it."""
|
|
112
|
+
return {
|
|
113
|
+
self.name(entity, fields): f"CREATE INDEX {quote(self.name(entity, fields))} ON {quote(entity)} "
|
|
114
|
+
f"({', '.join(self.engine.match_text(quote(field)) for field in fields)}, \"_key\")"
|
|
115
|
+
for entity, fields in self.keys()
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
def names(self) -> frozenset[str]:
|
|
119
|
+
return frozenset(self.by_name())
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Selection:
|
|
123
|
+
"""One read of ``entity`` through ``match``: :meth:`page` answers it."""
|
|
124
|
+
|
|
125
|
+
def __init__(self, store: WorldDatabase, entity: str, match: RowMatch) -> None:
|
|
126
|
+
self.store = store
|
|
127
|
+
self.entity = entity
|
|
128
|
+
self.match = match
|
|
129
|
+
properties = store.contract.entities[entity]["schema"].get("properties", {})
|
|
130
|
+
sent = [(arg, path) for arg, path in match.filters.items() if arg in match.values]
|
|
131
|
+
terms = [self._term(properties, arg, path) for arg, path in sent]
|
|
132
|
+
self.terms = [term for term in terms if term is not None]
|
|
133
|
+
self.exact = len(self.terms) == len(terms)
|
|
134
|
+
|
|
135
|
+
def _term(self, properties: dict, arg: str, path: str) -> tuple[str, list[str]] | None:
|
|
136
|
+
if self.match.modes.get(arg, "equals") != "equals":
|
|
137
|
+
return None
|
|
138
|
+
allowed = exact_field(properties, path)
|
|
139
|
+
sent = self.match.values[arg]
|
|
140
|
+
wanted = [str(value) for value in (sent if isinstance(sent, list) else [sent])]
|
|
141
|
+
if allowed is None or not all(allowed(value) for value in wanted):
|
|
142
|
+
return None
|
|
143
|
+
return path, [value.lower() for value in wanted]
|
|
144
|
+
|
|
145
|
+
def _where(self) -> tuple[str, list[str]]:
|
|
146
|
+
clauses = [
|
|
147
|
+
f"{self.store.match_text(quote(field))} IN ({', '.join('?' for _ in values)})"
|
|
148
|
+
for field, values in self.terms
|
|
149
|
+
]
|
|
150
|
+
params = [value for _, values in self.terms for value in values]
|
|
151
|
+
return (f" WHERE {' AND '.join(clauses)}" if clauses else ""), params
|
|
152
|
+
|
|
153
|
+
def statement(self, limit: int | None, offset: int) -> tuple[str, tuple]:
|
|
154
|
+
"""The engine's page of the exact terms, in key order."""
|
|
155
|
+
where, params = self._where()
|
|
156
|
+
clause, paging = self.store.page_clause(limit, offset)
|
|
157
|
+
return f"SELECT payload FROM {quote(self.entity)}{where} ORDER BY _key{clause}", (*params, *paging)
|
|
158
|
+
|
|
159
|
+
def page(self, limit: int | None, offset: int) -> tuple[list[dict], int]:
|
|
160
|
+
"""A page of the rows ``match`` selects and how many it selects in all."""
|
|
161
|
+
if any(not values for _, values in self.terms):
|
|
162
|
+
return [], 0
|
|
163
|
+
if self.exact:
|
|
164
|
+
where, params = self._where()
|
|
165
|
+
total = self.store.query(f"SELECT COUNT(*) FROM {quote(self.entity)}{where}", tuple(params))[0][0]
|
|
166
|
+
return [json.loads(row[0]) for row in self.store.query(*self.statement(limit, offset))], total
|
|
167
|
+
matched = self.match.select(json.loads(row[0]) for row in self.store.query(*self.statement(None, 0)))
|
|
168
|
+
end = None if limit is None else offset + limit
|
|
169
|
+
return matched[offset:end], len(matched)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""The row a world database answers a query with, whatever engine read it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Iterable, Sequence
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Row(tuple):
|
|
9
|
+
"""One result row: a tuple of its values that also answers by column name, so
|
|
10
|
+
``row[0]``, ``row["name"]``, ``dict(row)`` and ``tuple(row) == (...)`` hold on
|
|
11
|
+
every engine. A name is matched exactly, then case-insensitively; with two
|
|
12
|
+
columns of one name, the first answers."""
|
|
13
|
+
|
|
14
|
+
def __new__(cls, columns: Sequence[str], values: Iterable[Any]) -> "Row":
|
|
15
|
+
row = super().__new__(cls, values)
|
|
16
|
+
row._columns = tuple(columns)
|
|
17
|
+
return row
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def all(cls, description: Sequence[Sequence[Any]] | None, values: Iterable[Iterable[Any]]) -> "Rows":
|
|
21
|
+
"""Every row of a result, named by a DB-API ``cursor.description``."""
|
|
22
|
+
columns = tuple(column[0] for column in description or ())
|
|
23
|
+
return Rows(columns, (cls(columns, row) for row in values))
|
|
24
|
+
|
|
25
|
+
def keys(self) -> list[str]:
|
|
26
|
+
return list(self._columns)
|
|
27
|
+
|
|
28
|
+
def __getitem__(self, key: Any) -> Any:
|
|
29
|
+
if not isinstance(key, str):
|
|
30
|
+
return super().__getitem__(key)
|
|
31
|
+
return super().__getitem__(self._index(key))
|
|
32
|
+
|
|
33
|
+
def _index(self, name: str) -> int:
|
|
34
|
+
if name in self._columns:
|
|
35
|
+
return self._columns.index(name)
|
|
36
|
+
folded = [column.lower() for column in self._columns]
|
|
37
|
+
if name.lower() in folded:
|
|
38
|
+
return folded.index(name.lower())
|
|
39
|
+
raise IndexError(f"No column named {name}")
|
|
40
|
+
|
|
41
|
+
def __getnewargs__(self) -> tuple:
|
|
42
|
+
return (self._columns, tuple(self))
|
|
43
|
+
|
|
44
|
+
def __repr__(self) -> str:
|
|
45
|
+
return f"Row({dict(zip(self._columns, self))!r})"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Rows(list):
|
|
49
|
+
"""A query's rows, which also name the columns when there are none."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, columns: Sequence[str], rows: Iterable[Row]) -> None:
|
|
52
|
+
super().__init__(rows)
|
|
53
|
+
self.columns = tuple(columns)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""The statements that write the runtime tables a verifier reads: the active task,
|
|
2
|
+
the run's condition bindings and the turn events. A world database and a
|
|
3
|
+
mock-tools world's raw file run the same statements."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
Statement = tuple[str, tuple]
|
|
8
|
+
|
|
9
|
+
TASK_CONTEXT = "benchmark_runtime_context"
|
|
10
|
+
RUN_PARAMETERS = "benchmark_runtime_parameters"
|
|
11
|
+
CONVERSATION_EVENTS = "benchmark_conversation_events"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RuntimeRows:
|
|
15
|
+
@staticmethod
|
|
16
|
+
def task_context(scenario_id: str) -> list[Statement]:
|
|
17
|
+
return [
|
|
18
|
+
(f"DELETE FROM {TASK_CONTEXT}", ()),
|
|
19
|
+
(f"INSERT INTO {TASK_CONTEXT} (context_key, scenario_id) VALUES ('active_task', ?)", (scenario_id,)),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def run_parameters(parameters: dict[str, str]) -> list[Statement]:
|
|
24
|
+
return [(f"DELETE FROM {RUN_PARAMETERS}", ())] + [
|
|
25
|
+
(f"INSERT INTO {RUN_PARAMETERS} (key, value) VALUES (?, ?)", (key, parameters[key]))
|
|
26
|
+
for key in sorted(parameters)
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def conversation_event(
|
|
31
|
+
scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
|
|
32
|
+
) -> list[Statement]:
|
|
33
|
+
return [
|
|
34
|
+
(
|
|
35
|
+
f"INSERT INTO {CONVERSATION_EVENTS} "
|
|
36
|
+
"(scenario_id, user_turn_index, event_type, actor, name, payload_json) VALUES (?, ?, ?, ?, ?, ?)",
|
|
37
|
+
(scenario_id, user_turn_index, event_type, actor, name, payload_json),
|
|
38
|
+
)
|
|
39
|
+
]
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import sqlite3
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from contextlib import closing
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Callable, Iterable
|
|
11
|
+
|
|
12
|
+
from ..compiler import json_pointer_parts, quote, scalar_types
|
|
13
|
+
from ..validation import ContractError, fail
|
|
14
|
+
from .base import STORED_LOCK_QUERY, SideTable, WorldDatabase
|
|
15
|
+
from .budget import QueryBudget
|
|
16
|
+
from .guard import STAGING, Access, AccessKind
|
|
17
|
+
|
|
18
|
+
ACCESS_KINDS = {
|
|
19
|
+
sqlite3.SQLITE_SELECT: AccessKind.SELECT,
|
|
20
|
+
sqlite3.SQLITE_READ: AccessKind.READ,
|
|
21
|
+
sqlite3.SQLITE_FUNCTION: AccessKind.FUNCTION,
|
|
22
|
+
sqlite3.SQLITE_RECURSIVE: AccessKind.RECURSIVE,
|
|
23
|
+
sqlite3.SQLITE_INSERT: AccessKind.WRITE,
|
|
24
|
+
sqlite3.SQLITE_UPDATE: AccessKind.WRITE,
|
|
25
|
+
sqlite3.SQLITE_DELETE: AccessKind.WRITE,
|
|
26
|
+
sqlite3.SQLITE_CREATE_TEMP_TABLE: AccessKind.STAGE,
|
|
27
|
+
sqlite3.SQLITE_CREATE_TEMP_INDEX: AccessKind.STAGE,
|
|
28
|
+
sqlite3.SQLITE_CREATE_TEMP_VIEW: AccessKind.STAGE,
|
|
29
|
+
sqlite3.SQLITE_DROP_TEMP_TABLE: AccessKind.STAGE,
|
|
30
|
+
sqlite3.SQLITE_DROP_TEMP_INDEX: AccessKind.STAGE,
|
|
31
|
+
sqlite3.SQLITE_DROP_TEMP_VIEW: AccessKind.STAGE,
|
|
32
|
+
sqlite3.SQLITE_SAVEPOINT: AccessKind.SAVEPOINT,
|
|
33
|
+
}
|
|
34
|
+
TEMP_SCHEMA = "temp"
|
|
35
|
+
UNSAFE_FUNCTIONS = frozenset({"load_extension", "writefile", "readfile"})
|
|
36
|
+
#: Virtual-machine steps between deadline checks: frequent enough to stop within milliseconds.
|
|
37
|
+
PROGRESS_STEPS = 10_000
|
|
38
|
+
MISSING_TABLE = re.compile(r"no such table: (.+?)\s*$", re.MULTILINE)
|
|
39
|
+
CARRIED_TABLE = "_gateway_calls"
|
|
40
|
+
SNAPSHOT_STEP_PAGES = 256
|
|
41
|
+
SNAPSHOT_STEP_PAUSE_S = 0.001
|
|
42
|
+
KELVIN_SIGN = 8490
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def access(action: int, first: str | None, second: str | None, database: str | None) -> Access:
|
|
46
|
+
"""SQLite's authorizer names a called function in ``second`` and every other object in ``first``."""
|
|
47
|
+
kind = ACCESS_KINDS.get(action, AccessKind.OTHER)
|
|
48
|
+
target = second if kind is AccessKind.FUNCTION else first
|
|
49
|
+
column = second if kind is AccessKind.READ else None
|
|
50
|
+
return Access(kind, target, STAGING if database == TEMP_SCHEMA else database, column=column)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_SINGLE_QUOTED = re.compile(r"'(?:[^']|'')*'")
|
|
54
|
+
_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _without_comments(sql: str) -> str:
|
|
58
|
+
out: list[str] = []
|
|
59
|
+
i, n = 0, len(sql)
|
|
60
|
+
quoted: str | None = None
|
|
61
|
+
while i < n:
|
|
62
|
+
ch = sql[i]
|
|
63
|
+
if quoted is not None:
|
|
64
|
+
out.append(ch)
|
|
65
|
+
quoted = None if ch == quoted else quoted
|
|
66
|
+
i += 1
|
|
67
|
+
continue
|
|
68
|
+
if ch in ("'", '"', "`"):
|
|
69
|
+
quoted = ch
|
|
70
|
+
out.append(ch)
|
|
71
|
+
i += 1
|
|
72
|
+
continue
|
|
73
|
+
if sql.startswith("--", i):
|
|
74
|
+
end = sql.find("\n", i)
|
|
75
|
+
i = n if end == -1 else end
|
|
76
|
+
continue
|
|
77
|
+
if sql.startswith("/*", i):
|
|
78
|
+
end = sql.find("*/", i + 2)
|
|
79
|
+
i = n if end == -1 else end + 2
|
|
80
|
+
continue
|
|
81
|
+
out.append(ch)
|
|
82
|
+
i += 1
|
|
83
|
+
return "".join(out)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _append_statement(statements: list[str], text: str) -> None:
|
|
87
|
+
stripped = text.strip()
|
|
88
|
+
if _without_comments(stripped).strip(" \t\r\n;"):
|
|
89
|
+
statements.append(stripped)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _json_path(field: str) -> str:
|
|
93
|
+
parts = json_pointer_parts(field, field, code="key")
|
|
94
|
+
return "$" + "".join(f".{json.dumps(part, ensure_ascii=False)}" for part in parts)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def quote_sql(value: str) -> str:
|
|
98
|
+
return "'" + value.replace("'", "''") + "'"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _projection(field: str) -> str:
|
|
102
|
+
return f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", {quote_sql(_json_path(field))})) VIRTUAL'
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _existing_file(path: Path) -> sqlite3.Connection:
|
|
106
|
+
"""``mode=rw``: a missing file is refused, never created."""
|
|
107
|
+
return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=rw", uri=True)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class SQLiteWorldDatabase(WorldDatabase):
|
|
111
|
+
engine = "sqlite"
|
|
112
|
+
engine_label = "SQLite"
|
|
113
|
+
target = "sqlite-json/1"
|
|
114
|
+
snapshot_name = "state.sqlite"
|
|
115
|
+
content_type = "application/vnd.sqlite3"
|
|
116
|
+
native_error = sqlite3.Error
|
|
117
|
+
unsafe_functions = UNSAFE_FUNCTIONS
|
|
118
|
+
insertion_order = "rowid"
|
|
119
|
+
|
|
120
|
+
@classmethod
|
|
121
|
+
def ddl(cls, entities: dict) -> str:
|
|
122
|
+
"""The payload is the source of truth; fields are virtual columns so verifier SQL can name them and SQLite
|
|
123
|
+
enforces UNIQUE and foreign keys without affinity coercion."""
|
|
124
|
+
lines = [
|
|
125
|
+
"-- Generated by gateway-world-compiler/1; do not edit.",
|
|
126
|
+
"PRAGMA foreign_keys = ON;",
|
|
127
|
+
]
|
|
128
|
+
for name, entity in entities.items():
|
|
129
|
+
lines.append(f"CREATE TABLE {quote(name)} (\n " + ",\n ".join(cls._columns(entities, entity)) + "\n);")
|
|
130
|
+
return "\n".join(lines) + "\n"
|
|
131
|
+
|
|
132
|
+
@staticmethod
|
|
133
|
+
def _columns(entities: dict, entity: dict) -> list[str]:
|
|
134
|
+
properties = entity["schema"].get("properties", {})
|
|
135
|
+
columns = [
|
|
136
|
+
'"_key" TEXT PRIMARY KEY NOT NULL',
|
|
137
|
+
'"payload" TEXT NOT NULL CHECK(json_valid("payload"))',
|
|
138
|
+
]
|
|
139
|
+
columns.extend(_projection(field) for field, prop in sorted(properties.items()) if scalar_types(prop))
|
|
140
|
+
constrained = {field for fields in [entity["primary_key"], *entity["unique"]] for field in fields} | {
|
|
141
|
+
field for relationship in entity["relationships"] for field in relationship["fields"]
|
|
142
|
+
}
|
|
143
|
+
columns.extend(_projection(field) for field in sorted(constrained - set(properties)))
|
|
144
|
+
columns.extend(
|
|
145
|
+
f"UNIQUE ({', '.join(quote(f) for f in fields)})" for fields in [entity["primary_key"], *entity["unique"]]
|
|
146
|
+
)
|
|
147
|
+
for rel in entity["relationships"]:
|
|
148
|
+
target = entities[rel["target"]]
|
|
149
|
+
columns.append(
|
|
150
|
+
f"FOREIGN KEY ({', '.join(quote(f) for f in rel['fields'])}) "
|
|
151
|
+
f"REFERENCES {quote(rel['target'])} "
|
|
152
|
+
f"({', '.join(quote(f) for f in target['primary_key'])}) "
|
|
153
|
+
"ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"
|
|
154
|
+
)
|
|
155
|
+
return columns
|
|
156
|
+
|
|
157
|
+
@staticmethod
|
|
158
|
+
def split_statements(script: str) -> list[str]:
|
|
159
|
+
"""A ``;`` inside a literal or a trigger body never splits; pinned by ``sql-verifier-vectors.json``."""
|
|
160
|
+
statements: list[str] = []
|
|
161
|
+
buffer = ""
|
|
162
|
+
for line in script.splitlines(keepends=True):
|
|
163
|
+
buffer += line
|
|
164
|
+
if sqlite3.complete_statement(buffer):
|
|
165
|
+
_append_statement(statements, buffer)
|
|
166
|
+
buffer = ""
|
|
167
|
+
_append_statement(statements, buffer)
|
|
168
|
+
return statements
|
|
169
|
+
|
|
170
|
+
@staticmethod
|
|
171
|
+
def exceeds_limit(error: BaseException) -> bool:
|
|
172
|
+
return getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_TOOBIG
|
|
173
|
+
|
|
174
|
+
@staticmethod
|
|
175
|
+
def missing_table(error: BaseException) -> str | None:
|
|
176
|
+
match = MISSING_TABLE.search(str(error))
|
|
177
|
+
return match.group(1) if match else None
|
|
178
|
+
|
|
179
|
+
def _open_connection(self) -> None:
|
|
180
|
+
# cached_statements=0: a statement served from the cache is not re-authorized, so a text first
|
|
181
|
+
# prepared through one door would run unchecked through another.
|
|
182
|
+
self._conn = sqlite3.connect(
|
|
183
|
+
self.path,
|
|
184
|
+
isolation_level=None,
|
|
185
|
+
check_same_thread=False,
|
|
186
|
+
cached_statements=0,
|
|
187
|
+
)
|
|
188
|
+
self._conn.row_factory = sqlite3.Row
|
|
189
|
+
self._saved_limits: dict[int, int] = {}
|
|
190
|
+
|
|
191
|
+
def _enforce_strict_dialect(self) -> None:
|
|
192
|
+
self.forbid_double_quoted_literals(self._conn)
|
|
193
|
+
self._conn.execute("PRAGMA foreign_keys = ON")
|
|
194
|
+
if self._conn.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
|
|
195
|
+
fail("sqlite", "Foreign-key enforcement is required")
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
def forbid_double_quoted_literals(conn: Any) -> None:
|
|
199
|
+
"""SQLite reads a double-quoted missing column as a literal, so a verifier naming one would score 0 silently."""
|
|
200
|
+
if not hasattr(conn, "setconfig"):
|
|
201
|
+
fail(
|
|
202
|
+
"python_version",
|
|
203
|
+
f"Python {sys.version_info.major}.{sys.version_info.minor} cannot turn off SQLite's "
|
|
204
|
+
"double-quoted string literals; the world store needs Python 3.12 or newer.",
|
|
205
|
+
)
|
|
206
|
+
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
|
|
207
|
+
|
|
208
|
+
def _execute(self, sql: str, params: Any = ()) -> sqlite3.Cursor:
|
|
209
|
+
return self._conn.execute(sql, params)
|
|
210
|
+
|
|
211
|
+
def _executemany(self, sql: str, rows: Any) -> sqlite3.Cursor:
|
|
212
|
+
return self._conn.executemany(sql, rows)
|
|
213
|
+
|
|
214
|
+
def _execute_script(self, script: str) -> None:
|
|
215
|
+
self._conn.executescript(script)
|
|
216
|
+
|
|
217
|
+
def _schema_objects(self) -> dict[str, str]:
|
|
218
|
+
return self._master(self._conn)
|
|
219
|
+
|
|
220
|
+
def _objects_of(self, script: str) -> dict[str, str]:
|
|
221
|
+
with closing(sqlite3.connect(":memory:")) as fresh:
|
|
222
|
+
fresh.executescript(script)
|
|
223
|
+
return self._master(fresh)
|
|
224
|
+
|
|
225
|
+
@classmethod
|
|
226
|
+
def match_text(cls, column: str) -> str:
|
|
227
|
+
# SQLite's lower() folds ASCII only; the one other character Python lowers to ASCII is the
|
|
228
|
+
# KELVIN SIGN ('K' -> 'k'), so it is folded first and an ASCII comparison equals RowMatch's.
|
|
229
|
+
return f"lower(replace(CAST({column} AS TEXT), char({KELVIN_SIGN}), 'k'))"
|
|
230
|
+
|
|
231
|
+
@staticmethod
|
|
232
|
+
def _master(conn: sqlite3.Connection) -> dict[str, str]:
|
|
233
|
+
return dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL"))
|
|
234
|
+
|
|
235
|
+
def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
|
|
236
|
+
self._conn.create_function("gateway_key", 2, function, deterministic=True)
|
|
237
|
+
|
|
238
|
+
def _install_guard(self) -> None:
|
|
239
|
+
self._conn.set_authorizer(self._authorize)
|
|
240
|
+
|
|
241
|
+
def _authorize(self, action, first, second, database, trigger):
|
|
242
|
+
if self._guard_allows(access(action, first, second, database)):
|
|
243
|
+
return sqlite3.SQLITE_OK
|
|
244
|
+
return sqlite3.SQLITE_DENY
|
|
245
|
+
|
|
246
|
+
def _begin_write(self) -> None:
|
|
247
|
+
self._conn.execute("BEGIN IMMEDIATE")
|
|
248
|
+
|
|
249
|
+
def _commit(self) -> None:
|
|
250
|
+
self._conn.execute("COMMIT")
|
|
251
|
+
|
|
252
|
+
def _rollback(self) -> None:
|
|
253
|
+
self._conn.execute("ROLLBACK")
|
|
254
|
+
|
|
255
|
+
def _savepoint(self, name: str) -> None:
|
|
256
|
+
self._conn.execute(f"SAVEPOINT {name}")
|
|
257
|
+
|
|
258
|
+
def _release(self, name: str) -> None:
|
|
259
|
+
self._conn.execute(f"RELEASE {name}")
|
|
260
|
+
|
|
261
|
+
def _rollback_to(self, name: str) -> None:
|
|
262
|
+
self._conn.execute(f"ROLLBACK TO {name}")
|
|
263
|
+
|
|
264
|
+
def page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
|
|
265
|
+
return " LIMIT ? OFFSET ?", (-1 if limit is None else limit, offset)
|
|
266
|
+
|
|
267
|
+
def _relationships_broken(self) -> bool:
|
|
268
|
+
return self._conn.execute("PRAGMA foreign_key_check").fetchone() is not None
|
|
269
|
+
|
|
270
|
+
def _is_constraint_violation(self, error: BaseException) -> bool:
|
|
271
|
+
return isinstance(error, sqlite3.IntegrityError)
|
|
272
|
+
|
|
273
|
+
def _drop_staging(self) -> None:
|
|
274
|
+
objects = self._conn.execute(
|
|
275
|
+
"SELECT type, name FROM sqlite_temp_master WHERE type IN ('view', 'table') ORDER BY type DESC"
|
|
276
|
+
).fetchall()
|
|
277
|
+
for kind, name in objects:
|
|
278
|
+
self._conn.execute(f"DROP {kind} IF EXISTS temp.{quote(name)}")
|
|
279
|
+
|
|
280
|
+
def _attach_file(self, name: str, path: str | Path) -> None:
|
|
281
|
+
self._conn.execute("ATTACH DATABASE ? AS ?", (str(path), name))
|
|
282
|
+
|
|
283
|
+
def _detach_file(self, name: str) -> None:
|
|
284
|
+
self._conn.execute("DETACH DATABASE ?", (name,))
|
|
285
|
+
|
|
286
|
+
@staticmethod
|
|
287
|
+
def carry_entries(path: Path, entries: list[str]) -> None:
|
|
288
|
+
with closing(_existing_file(path)) as connection, connection:
|
|
289
|
+
connection.execute(f"CREATE TABLE {CARRIED_TABLE} (ordinal INTEGER PRIMARY KEY, entry TEXT NOT NULL)")
|
|
290
|
+
connection.executemany(f"INSERT INTO {CARRIED_TABLE} VALUES (?, ?)", list(enumerate(entries)))
|
|
291
|
+
|
|
292
|
+
@staticmethod
|
|
293
|
+
def identifiers(sql: str) -> frozenset[str]:
|
|
294
|
+
bare = _SINGLE_QUOTED.sub("''", _without_comments(sql))
|
|
295
|
+
return frozenset(name.lower() for name in _IDENTIFIER.findall(bare))
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def write_side_tables(path: Path, tables: Iterable[SideTable]) -> None:
|
|
299
|
+
with closing(sqlite3.connect(path)) as connection, connection:
|
|
300
|
+
for table in tables:
|
|
301
|
+
columns = ", ".join(f"{quote(column)} {kind}" for column, kind in table.columns)
|
|
302
|
+
connection.execute(f"CREATE TABLE {quote(table.name)} ({columns})")
|
|
303
|
+
marks = ", ".join("?" for _ in table.columns)
|
|
304
|
+
connection.executemany(f"INSERT INTO {quote(table.name)} VALUES ({marks})", table.rows)
|
|
305
|
+
|
|
306
|
+
@staticmethod
|
|
307
|
+
def take_entries(path: Path) -> list[str]:
|
|
308
|
+
with closing(_existing_file(path)) as connection, connection:
|
|
309
|
+
carried = connection.execute(
|
|
310
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (CARRIED_TABLE,)
|
|
311
|
+
).fetchone()
|
|
312
|
+
if carried is None:
|
|
313
|
+
return []
|
|
314
|
+
entries = [entry for (entry,) in connection.execute(f"SELECT entry FROM {CARRIED_TABLE} ORDER BY ordinal")]
|
|
315
|
+
connection.execute(f"DROP TABLE {CARRIED_TABLE}")
|
|
316
|
+
return entries
|
|
317
|
+
|
|
318
|
+
@staticmethod
|
|
319
|
+
def drop_indexes(path: Path, names: Iterable[str]) -> None:
|
|
320
|
+
with closing(_existing_file(path)) as connection, connection:
|
|
321
|
+
held = {name for (name,) in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")}
|
|
322
|
+
for name in sorted(set(names) & held):
|
|
323
|
+
connection.execute(f"DROP INDEX {quote(name)}")
|
|
324
|
+
|
|
325
|
+
@staticmethod
|
|
326
|
+
def recover_copy(source: Path, destination: Path) -> None:
|
|
327
|
+
# Opening the file read-write is what rolls back a hot journal its dead writer left beside it.
|
|
328
|
+
with closing(_existing_file(source)) as live, closing(sqlite3.connect(destination)) as copy:
|
|
329
|
+
live.backup(copy)
|
|
330
|
+
|
|
331
|
+
@staticmethod
|
|
332
|
+
def stored_lock(path: Path) -> dict:
|
|
333
|
+
connection = sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
|
|
334
|
+
try:
|
|
335
|
+
rows = connection.execute(STORED_LOCK_QUERY).fetchall()
|
|
336
|
+
except sqlite3.Error as error:
|
|
337
|
+
raise ContractError("state_contract", "State has no compatible schema contract") from error
|
|
338
|
+
finally:
|
|
339
|
+
connection.close()
|
|
340
|
+
return WorldDatabase.lock_from(rows)
|
|
341
|
+
|
|
342
|
+
@staticmethod
|
|
343
|
+
def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
|
|
344
|
+
connection = sqlite3.connect(destination, isolation_level=None)
|
|
345
|
+
try:
|
|
346
|
+
connection.execute("PRAGMA foreign_keys = ON")
|
|
347
|
+
connection.execute("ATTACH DATABASE ? AS stored", (str(source),))
|
|
348
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
349
|
+
for table in ledger:
|
|
350
|
+
connection.execute(f"INSERT INTO main.{quote(table)} SELECT * FROM stored.{quote(table)}")
|
|
351
|
+
for entity in entities:
|
|
352
|
+
connection.execute(
|
|
353
|
+
f"INSERT INTO main.{quote(entity)} (_key, payload) SELECT _key, payload FROM stored.{quote(entity)}"
|
|
354
|
+
)
|
|
355
|
+
connection.execute("COMMIT")
|
|
356
|
+
except sqlite3.Error as error:
|
|
357
|
+
raise ContractError("state_contract", f"The stored rows could not be migrated ({error})") from None
|
|
358
|
+
finally:
|
|
359
|
+
connection.close()
|
|
360
|
+
|
|
361
|
+
def _write_copy(self, destination: Path) -> None:
|
|
362
|
+
"""Between steps the lock is let go for a waiting request; the backup runs on this connection, so a write made
|
|
363
|
+
then lands in the copy."""
|
|
364
|
+
target = sqlite3.connect(destination)
|
|
365
|
+
try:
|
|
366
|
+
with self._lock:
|
|
367
|
+
self._conn.backup(target, pages=SNAPSHOT_STEP_PAGES, progress=self._between_steps)
|
|
368
|
+
finally:
|
|
369
|
+
target.close()
|
|
370
|
+
|
|
371
|
+
def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
|
|
372
|
+
if budget is None:
|
|
373
|
+
self._conn.set_progress_handler(None, 0)
|
|
374
|
+
for limit, value in self._saved_limits.items():
|
|
375
|
+
self._conn.setlimit(limit, value)
|
|
376
|
+
self._saved_limits = {}
|
|
377
|
+
return
|
|
378
|
+
self._saved_limits[sqlite3.SQLITE_LIMIT_LENGTH] = self._conn.setlimit(
|
|
379
|
+
sqlite3.SQLITE_LIMIT_LENGTH, budget.value_bytes
|
|
380
|
+
)
|
|
381
|
+
self._conn.set_progress_handler(lambda: int(time.monotonic() >= deadline), PROGRESS_STEPS)
|
|
382
|
+
|
|
383
|
+
def _between_steps(self, _status: int, remaining: int, _total: int) -> None:
|
|
384
|
+
if not remaining:
|
|
385
|
+
return
|
|
386
|
+
self._lock.release()
|
|
387
|
+
try:
|
|
388
|
+
time.sleep(SNAPSHOT_STEP_PAUSE_S)
|
|
389
|
+
finally:
|
|
390
|
+
self._lock.acquire()
|
|
391
|
+
|
|
392
|
+
def _close_connection(self) -> None:
|
|
393
|
+
self._conn.close()
|