@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,169 @@
1
+ """A filtered read of one entity, answered by the engine wherever it answers exactly.
2
+
3
+ :class:`Selection` splits a :class:`RowMatch` into the terms the engine compares
4
+ exactly as ``RowMatch`` does (equality on a top-level field typed string alone,
5
+ or integer alone with values that fit 64 bits, every value ASCII) and the rest.
6
+ The exact terms run in SQL over the field's column, counted and paged by the
7
+ engine; any other term is checked by ``RowMatch`` over the rows they left. Either
8
+ way the answer is ``RowMatch`` over every row, in key order.
9
+
10
+ :class:`ReadIndexes` are what make those terms a search: one per primary key,
11
+ unique key and relationship whose fields are all such fields, and one per field a
12
+ version's declared routes filter on, over each field's match text and then
13
+ ``_key``, so an equality read finds its rows in key order and stops at its page.
14
+ They are not the contract's DDL: a store opens with or without them (each one
15
+ only exactly as its name derives it), a snapshot is written without them, and a
16
+ host builds them into the files its sessions read (the version's template, a
17
+ restored session).
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ import re
24
+ from typing import TYPE_CHECKING, Callable, Iterable
25
+
26
+ from ..compiler import quote, scalar_types
27
+ from .matching import RowMatch
28
+
29
+ if TYPE_CHECKING:
30
+ from .base import WorldDatabase
31
+
32
+ FORMAT = "1"
33
+ PREFIX = "_gateway_match"
34
+ INT64 = range(-(2**63), 2**63)
35
+ _INTEGER = re.compile(r"-?[0-9]+")
36
+
37
+
38
+ def _int64_text(value: str) -> bool:
39
+ return value.isascii() and _INTEGER.fullmatch(value) is not None and int(value) in INT64
40
+
41
+
42
+ #: The field types whose stored text the engine's match text spells as Python's
43
+ #: ``str`` does, by type, with the values a term may compare exactly.
44
+ EXACT_VALUES: dict[frozenset[str], Callable[[str], bool]] = {
45
+ frozenset({"string"}): str.isascii,
46
+ frozenset({"integer"}): _int64_text,
47
+ }
48
+
49
+
50
+ def _three(parts: list[str]) -> bool:
51
+ return len(parts) == 3
52
+
53
+
54
+ def _field_sets(entity: dict) -> list[list[str]]:
55
+ return [entity["primary_key"], *entity["unique"], *(rel["fields"] for rel in entity["relationships"])]
56
+
57
+
58
+ def exact_field(properties: dict, field: str) -> Callable[[str], bool] | None:
59
+ """How a value sent for ``field`` is checked before the engine compares it, or
60
+ None when the engine cannot compare that field exactly (a dotted path, an
61
+ array, an object, a boolean, a number, a mix of types)."""
62
+ prop = properties.get(field) if "." not in field else None
63
+ if not isinstance(prop, dict):
64
+ return None
65
+ return EXACT_VALUES.get(frozenset(scalar_types(prop)))
66
+
67
+
68
+ Fields = tuple[str, ...]
69
+
70
+
71
+ class ReadIndexes:
72
+ """The read indexes of one store's contract, as the store's engine writes them:
73
+ its keys and relationships, and ``extra`` (entity, fields) a version's routes
74
+ filter on."""
75
+
76
+ def __init__(self, engine: type[WorldDatabase], entities: dict, extra: Iterable[tuple[str, Fields]] = ()) -> None:
77
+ self.engine = engine
78
+ self.entities = entities
79
+ self.extra = list(extra)
80
+
81
+ @classmethod
82
+ def of(cls, store: WorldDatabase, extra: Iterable[tuple[str, Fields]] = ()) -> ReadIndexes:
83
+ return cls(type(store), store.contract.entities, extra)
84
+
85
+ @staticmethod
86
+ def named(names: Iterable[str]) -> list[str]:
87
+ """The read index names among ``names``."""
88
+ return [name for name in names if name.split(":", 1)[0] == PREFIX]
89
+
90
+ @staticmethod
91
+ def parse(names: Iterable[str]) -> list[tuple[str, Fields]]:
92
+ """The (entity, fields) each read index name says it covers; another name says none."""
93
+ parts = [name.split(":", 2) for name in ReadIndexes.named(names)]
94
+ return [(entity, tuple(fields.split(","))) for _, entity, fields in filter(_three, parts)]
95
+
96
+ def keys(self) -> list[tuple[str, Fields]]:
97
+ """Every (entity, fields) a read index covers, each once, the fields all exact."""
98
+ declared = [(name, tuple(fields)) for name, entity in self.entities.items() for fields in _field_sets(entity)]
99
+ return list(dict.fromkeys(key for key in [*declared, *self.extra] if self._exact(*key)))
100
+
101
+ def _exact(self, entity: str, fields: Fields) -> bool:
102
+ declared = self.entities.get(entity)
103
+ properties = declared["schema"].get("properties", {}) if declared else {}
104
+ return bool(fields) and all(exact_field(properties, field) is not None for field in fields)
105
+
106
+ @staticmethod
107
+ def name(entity: str, fields: tuple[str, ...]) -> str:
108
+ return f"{PREFIX}:{entity}:{','.join(fields)}"
109
+
110
+ def by_name(self) -> dict[str, str]:
111
+ """Each read index's name and the statement that builds it."""
112
+ return {
113
+ self.name(entity, fields): f"CREATE INDEX {quote(self.name(entity, fields))} ON {quote(entity)} "
114
+ f"({', '.join(self.engine.match_text(quote(field)) for field in fields)}, \"_key\")"
115
+ for entity, fields in self.keys()
116
+ }
117
+
118
+ def names(self) -> frozenset[str]:
119
+ return frozenset(self.by_name())
120
+
121
+
122
+ class Selection:
123
+ """One read of ``entity`` through ``match``: :meth:`page` answers it."""
124
+
125
+ def __init__(self, store: WorldDatabase, entity: str, match: RowMatch) -> None:
126
+ self.store = store
127
+ self.entity = entity
128
+ self.match = match
129
+ properties = store.contract.entities[entity]["schema"].get("properties", {})
130
+ sent = [(arg, path) for arg, path in match.filters.items() if arg in match.values]
131
+ terms = [self._term(properties, arg, path) for arg, path in sent]
132
+ self.terms = [term for term in terms if term is not None]
133
+ self.exact = len(self.terms) == len(terms)
134
+
135
+ def _term(self, properties: dict, arg: str, path: str) -> tuple[str, list[str]] | None:
136
+ if self.match.modes.get(arg, "equals") != "equals":
137
+ return None
138
+ allowed = exact_field(properties, path)
139
+ sent = self.match.values[arg]
140
+ wanted = [str(value) for value in (sent if isinstance(sent, list) else [sent])]
141
+ if allowed is None or not all(allowed(value) for value in wanted):
142
+ return None
143
+ return path, [value.lower() for value in wanted]
144
+
145
+ def _where(self) -> tuple[str, list[str]]:
146
+ clauses = [
147
+ f"{self.store.match_text(quote(field))} IN ({', '.join('?' for _ in values)})"
148
+ for field, values in self.terms
149
+ ]
150
+ params = [value for _, values in self.terms for value in values]
151
+ return (f" WHERE {' AND '.join(clauses)}" if clauses else ""), params
152
+
153
+ def statement(self, limit: int | None, offset: int) -> tuple[str, tuple]:
154
+ """The engine's page of the exact terms, in key order."""
155
+ where, params = self._where()
156
+ clause, paging = self.store.page_clause(limit, offset)
157
+ return f"SELECT payload FROM {quote(self.entity)}{where} ORDER BY _key{clause}", (*params, *paging)
158
+
159
+ def page(self, limit: int | None, offset: int) -> tuple[list[dict], int]:
160
+ """A page of the rows ``match`` selects and how many it selects in all."""
161
+ if any(not values for _, values in self.terms):
162
+ return [], 0
163
+ if self.exact:
164
+ where, params = self._where()
165
+ total = self.store.query(f"SELECT COUNT(*) FROM {quote(self.entity)}{where}", tuple(params))[0][0]
166
+ return [json.loads(row[0]) for row in self.store.query(*self.statement(limit, offset))], total
167
+ matched = self.match.select(json.loads(row[0]) for row in self.store.query(*self.statement(None, 0)))
168
+ end = None if limit is None else offset + limit
169
+ return matched[offset:end], len(matched)
@@ -0,0 +1,53 @@
1
+ """The row a world database answers a query with, whatever engine read it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Iterable, Sequence
6
+
7
+
8
+ class Row(tuple):
9
+ """One result row: a tuple of its values that also answers by column name, so
10
+ ``row[0]``, ``row["name"]``, ``dict(row)`` and ``tuple(row) == (...)`` hold on
11
+ every engine. A name is matched exactly, then case-insensitively; with two
12
+ columns of one name, the first answers."""
13
+
14
+ def __new__(cls, columns: Sequence[str], values: Iterable[Any]) -> "Row":
15
+ row = super().__new__(cls, values)
16
+ row._columns = tuple(columns)
17
+ return row
18
+
19
+ @classmethod
20
+ def all(cls, description: Sequence[Sequence[Any]] | None, values: Iterable[Iterable[Any]]) -> "Rows":
21
+ """Every row of a result, named by a DB-API ``cursor.description``."""
22
+ columns = tuple(column[0] for column in description or ())
23
+ return Rows(columns, (cls(columns, row) for row in values))
24
+
25
+ def keys(self) -> list[str]:
26
+ return list(self._columns)
27
+
28
+ def __getitem__(self, key: Any) -> Any:
29
+ if not isinstance(key, str):
30
+ return super().__getitem__(key)
31
+ return super().__getitem__(self._index(key))
32
+
33
+ def _index(self, name: str) -> int:
34
+ if name in self._columns:
35
+ return self._columns.index(name)
36
+ folded = [column.lower() for column in self._columns]
37
+ if name.lower() in folded:
38
+ return folded.index(name.lower())
39
+ raise IndexError(f"No column named {name}")
40
+
41
+ def __getnewargs__(self) -> tuple:
42
+ return (self._columns, tuple(self))
43
+
44
+ def __repr__(self) -> str:
45
+ return f"Row({dict(zip(self._columns, self))!r})"
46
+
47
+
48
+ class Rows(list):
49
+ """A query's rows, which also name the columns when there are none."""
50
+
51
+ def __init__(self, columns: Sequence[str], rows: Iterable[Row]) -> None:
52
+ super().__init__(rows)
53
+ self.columns = tuple(columns)
@@ -0,0 +1,39 @@
1
+ """The statements that write the runtime tables a verifier reads: the active task,
2
+ the run's condition bindings and the turn events. A world database and a
3
+ mock-tools world's raw file run the same statements."""
4
+
5
+ from __future__ import annotations
6
+
7
+ Statement = tuple[str, tuple]
8
+
9
+ TASK_CONTEXT = "benchmark_runtime_context"
10
+ RUN_PARAMETERS = "benchmark_runtime_parameters"
11
+ CONVERSATION_EVENTS = "benchmark_conversation_events"
12
+
13
+
14
+ class RuntimeRows:
15
+ @staticmethod
16
+ def task_context(scenario_id: str) -> list[Statement]:
17
+ return [
18
+ (f"DELETE FROM {TASK_CONTEXT}", ()),
19
+ (f"INSERT INTO {TASK_CONTEXT} (context_key, scenario_id) VALUES ('active_task', ?)", (scenario_id,)),
20
+ ]
21
+
22
+ @staticmethod
23
+ def run_parameters(parameters: dict[str, str]) -> list[Statement]:
24
+ return [(f"DELETE FROM {RUN_PARAMETERS}", ())] + [
25
+ (f"INSERT INTO {RUN_PARAMETERS} (key, value) VALUES (?, ?)", (key, parameters[key]))
26
+ for key in sorted(parameters)
27
+ ]
28
+
29
+ @staticmethod
30
+ def conversation_event(
31
+ scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
32
+ ) -> list[Statement]:
33
+ return [
34
+ (
35
+ f"INSERT INTO {CONVERSATION_EVENTS} "
36
+ "(scenario_id, user_turn_index, event_type, actor, name, payload_json) VALUES (?, ?, ?, ?, ?, ?)",
37
+ (scenario_id, user_turn_index, event_type, actor, name, payload_json),
38
+ )
39
+ ]
@@ -0,0 +1,393 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import re
5
+ import sqlite3
6
+ import sys
7
+ import time
8
+ from contextlib import closing
9
+ from pathlib import Path
10
+ from typing import Any, Callable, Iterable
11
+
12
+ from ..compiler import json_pointer_parts, quote, scalar_types
13
+ from ..validation import ContractError, fail
14
+ from .base import STORED_LOCK_QUERY, SideTable, WorldDatabase
15
+ from .budget import QueryBudget
16
+ from .guard import STAGING, Access, AccessKind
17
+
18
+ ACCESS_KINDS = {
19
+ sqlite3.SQLITE_SELECT: AccessKind.SELECT,
20
+ sqlite3.SQLITE_READ: AccessKind.READ,
21
+ sqlite3.SQLITE_FUNCTION: AccessKind.FUNCTION,
22
+ sqlite3.SQLITE_RECURSIVE: AccessKind.RECURSIVE,
23
+ sqlite3.SQLITE_INSERT: AccessKind.WRITE,
24
+ sqlite3.SQLITE_UPDATE: AccessKind.WRITE,
25
+ sqlite3.SQLITE_DELETE: AccessKind.WRITE,
26
+ sqlite3.SQLITE_CREATE_TEMP_TABLE: AccessKind.STAGE,
27
+ sqlite3.SQLITE_CREATE_TEMP_INDEX: AccessKind.STAGE,
28
+ sqlite3.SQLITE_CREATE_TEMP_VIEW: AccessKind.STAGE,
29
+ sqlite3.SQLITE_DROP_TEMP_TABLE: AccessKind.STAGE,
30
+ sqlite3.SQLITE_DROP_TEMP_INDEX: AccessKind.STAGE,
31
+ sqlite3.SQLITE_DROP_TEMP_VIEW: AccessKind.STAGE,
32
+ sqlite3.SQLITE_SAVEPOINT: AccessKind.SAVEPOINT,
33
+ }
34
+ TEMP_SCHEMA = "temp"
35
+ UNSAFE_FUNCTIONS = frozenset({"load_extension", "writefile", "readfile"})
36
+ #: Virtual-machine steps between deadline checks: frequent enough to stop within milliseconds.
37
+ PROGRESS_STEPS = 10_000
38
+ MISSING_TABLE = re.compile(r"no such table: (.+?)\s*$", re.MULTILINE)
39
+ CARRIED_TABLE = "_gateway_calls"
40
+ SNAPSHOT_STEP_PAGES = 256
41
+ SNAPSHOT_STEP_PAUSE_S = 0.001
42
+ KELVIN_SIGN = 8490
43
+
44
+
45
+ def access(action: int, first: str | None, second: str | None, database: str | None) -> Access:
46
+ """SQLite's authorizer names a called function in ``second`` and every other object in ``first``."""
47
+ kind = ACCESS_KINDS.get(action, AccessKind.OTHER)
48
+ target = second if kind is AccessKind.FUNCTION else first
49
+ column = second if kind is AccessKind.READ else None
50
+ return Access(kind, target, STAGING if database == TEMP_SCHEMA else database, column=column)
51
+
52
+
53
+ _SINGLE_QUOTED = re.compile(r"'(?:[^']|'')*'")
54
+ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
55
+
56
+
57
+ def _without_comments(sql: str) -> str:
58
+ out: list[str] = []
59
+ i, n = 0, len(sql)
60
+ quoted: str | None = None
61
+ while i < n:
62
+ ch = sql[i]
63
+ if quoted is not None:
64
+ out.append(ch)
65
+ quoted = None if ch == quoted else quoted
66
+ i += 1
67
+ continue
68
+ if ch in ("'", '"', "`"):
69
+ quoted = ch
70
+ out.append(ch)
71
+ i += 1
72
+ continue
73
+ if sql.startswith("--", i):
74
+ end = sql.find("\n", i)
75
+ i = n if end == -1 else end
76
+ continue
77
+ if sql.startswith("/*", i):
78
+ end = sql.find("*/", i + 2)
79
+ i = n if end == -1 else end + 2
80
+ continue
81
+ out.append(ch)
82
+ i += 1
83
+ return "".join(out)
84
+
85
+
86
+ def _append_statement(statements: list[str], text: str) -> None:
87
+ stripped = text.strip()
88
+ if _without_comments(stripped).strip(" \t\r\n;"):
89
+ statements.append(stripped)
90
+
91
+
92
+ def _json_path(field: str) -> str:
93
+ parts = json_pointer_parts(field, field, code="key")
94
+ return "$" + "".join(f".{json.dumps(part, ensure_ascii=False)}" for part in parts)
95
+
96
+
97
+ def quote_sql(value: str) -> str:
98
+ return "'" + value.replace("'", "''") + "'"
99
+
100
+
101
+ def _projection(field: str) -> str:
102
+ return f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", {quote_sql(_json_path(field))})) VIRTUAL'
103
+
104
+
105
+ def _existing_file(path: Path) -> sqlite3.Connection:
106
+ """``mode=rw``: a missing file is refused, never created."""
107
+ return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=rw", uri=True)
108
+
109
+
110
+ class SQLiteWorldDatabase(WorldDatabase):
111
+ engine = "sqlite"
112
+ engine_label = "SQLite"
113
+ target = "sqlite-json/1"
114
+ snapshot_name = "state.sqlite"
115
+ content_type = "application/vnd.sqlite3"
116
+ native_error = sqlite3.Error
117
+ unsafe_functions = UNSAFE_FUNCTIONS
118
+ insertion_order = "rowid"
119
+
120
+ @classmethod
121
+ def ddl(cls, entities: dict) -> str:
122
+ """The payload is the source of truth; fields are virtual columns so verifier SQL can name them and SQLite
123
+ enforces UNIQUE and foreign keys without affinity coercion."""
124
+ lines = [
125
+ "-- Generated by gateway-world-compiler/1; do not edit.",
126
+ "PRAGMA foreign_keys = ON;",
127
+ ]
128
+ for name, entity in entities.items():
129
+ lines.append(f"CREATE TABLE {quote(name)} (\n " + ",\n ".join(cls._columns(entities, entity)) + "\n);")
130
+ return "\n".join(lines) + "\n"
131
+
132
+ @staticmethod
133
+ def _columns(entities: dict, entity: dict) -> list[str]:
134
+ properties = entity["schema"].get("properties", {})
135
+ columns = [
136
+ '"_key" TEXT PRIMARY KEY NOT NULL',
137
+ '"payload" TEXT NOT NULL CHECK(json_valid("payload"))',
138
+ ]
139
+ columns.extend(_projection(field) for field, prop in sorted(properties.items()) if scalar_types(prop))
140
+ constrained = {field for fields in [entity["primary_key"], *entity["unique"]] for field in fields} | {
141
+ field for relationship in entity["relationships"] for field in relationship["fields"]
142
+ }
143
+ columns.extend(_projection(field) for field in sorted(constrained - set(properties)))
144
+ columns.extend(
145
+ f"UNIQUE ({', '.join(quote(f) for f in fields)})" for fields in [entity["primary_key"], *entity["unique"]]
146
+ )
147
+ for rel in entity["relationships"]:
148
+ target = entities[rel["target"]]
149
+ columns.append(
150
+ f"FOREIGN KEY ({', '.join(quote(f) for f in rel['fields'])}) "
151
+ f"REFERENCES {quote(rel['target'])} "
152
+ f"({', '.join(quote(f) for f in target['primary_key'])}) "
153
+ "ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"
154
+ )
155
+ return columns
156
+
157
+ @staticmethod
158
+ def split_statements(script: str) -> list[str]:
159
+ """A ``;`` inside a literal or a trigger body never splits; pinned by ``sql-verifier-vectors.json``."""
160
+ statements: list[str] = []
161
+ buffer = ""
162
+ for line in script.splitlines(keepends=True):
163
+ buffer += line
164
+ if sqlite3.complete_statement(buffer):
165
+ _append_statement(statements, buffer)
166
+ buffer = ""
167
+ _append_statement(statements, buffer)
168
+ return statements
169
+
170
+ @staticmethod
171
+ def exceeds_limit(error: BaseException) -> bool:
172
+ return getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_TOOBIG
173
+
174
+ @staticmethod
175
+ def missing_table(error: BaseException) -> str | None:
176
+ match = MISSING_TABLE.search(str(error))
177
+ return match.group(1) if match else None
178
+
179
+ def _open_connection(self) -> None:
180
+ # cached_statements=0: a statement served from the cache is not re-authorized, so a text first
181
+ # prepared through one door would run unchecked through another.
182
+ self._conn = sqlite3.connect(
183
+ self.path,
184
+ isolation_level=None,
185
+ check_same_thread=False,
186
+ cached_statements=0,
187
+ )
188
+ self._conn.row_factory = sqlite3.Row
189
+ self._saved_limits: dict[int, int] = {}
190
+
191
+ def _enforce_strict_dialect(self) -> None:
192
+ self.forbid_double_quoted_literals(self._conn)
193
+ self._conn.execute("PRAGMA foreign_keys = ON")
194
+ if self._conn.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
195
+ fail("sqlite", "Foreign-key enforcement is required")
196
+
197
+ @staticmethod
198
+ def forbid_double_quoted_literals(conn: Any) -> None:
199
+ """SQLite reads a double-quoted missing column as a literal, so a verifier naming one would score 0 silently."""
200
+ if not hasattr(conn, "setconfig"):
201
+ fail(
202
+ "python_version",
203
+ f"Python {sys.version_info.major}.{sys.version_info.minor} cannot turn off SQLite's "
204
+ "double-quoted string literals; the world store needs Python 3.12 or newer.",
205
+ )
206
+ conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
207
+
208
+ def _execute(self, sql: str, params: Any = ()) -> sqlite3.Cursor:
209
+ return self._conn.execute(sql, params)
210
+
211
+ def _executemany(self, sql: str, rows: Any) -> sqlite3.Cursor:
212
+ return self._conn.executemany(sql, rows)
213
+
214
+ def _execute_script(self, script: str) -> None:
215
+ self._conn.executescript(script)
216
+
217
+ def _schema_objects(self) -> dict[str, str]:
218
+ return self._master(self._conn)
219
+
220
+ def _objects_of(self, script: str) -> dict[str, str]:
221
+ with closing(sqlite3.connect(":memory:")) as fresh:
222
+ fresh.executescript(script)
223
+ return self._master(fresh)
224
+
225
+ @classmethod
226
+ def match_text(cls, column: str) -> str:
227
+ # SQLite's lower() folds ASCII only; the one other character Python lowers to ASCII is the
228
+ # KELVIN SIGN ('K' -> 'k'), so it is folded first and an ASCII comparison equals RowMatch's.
229
+ return f"lower(replace(CAST({column} AS TEXT), char({KELVIN_SIGN}), 'k'))"
230
+
231
+ @staticmethod
232
+ def _master(conn: sqlite3.Connection) -> dict[str, str]:
233
+ return dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL"))
234
+
235
+ def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
236
+ self._conn.create_function("gateway_key", 2, function, deterministic=True)
237
+
238
+ def _install_guard(self) -> None:
239
+ self._conn.set_authorizer(self._authorize)
240
+
241
+ def _authorize(self, action, first, second, database, trigger):
242
+ if self._guard_allows(access(action, first, second, database)):
243
+ return sqlite3.SQLITE_OK
244
+ return sqlite3.SQLITE_DENY
245
+
246
+ def _begin_write(self) -> None:
247
+ self._conn.execute("BEGIN IMMEDIATE")
248
+
249
+ def _commit(self) -> None:
250
+ self._conn.execute("COMMIT")
251
+
252
+ def _rollback(self) -> None:
253
+ self._conn.execute("ROLLBACK")
254
+
255
+ def _savepoint(self, name: str) -> None:
256
+ self._conn.execute(f"SAVEPOINT {name}")
257
+
258
+ def _release(self, name: str) -> None:
259
+ self._conn.execute(f"RELEASE {name}")
260
+
261
+ def _rollback_to(self, name: str) -> None:
262
+ self._conn.execute(f"ROLLBACK TO {name}")
263
+
264
+ def page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
265
+ return " LIMIT ? OFFSET ?", (-1 if limit is None else limit, offset)
266
+
267
+ def _relationships_broken(self) -> bool:
268
+ return self._conn.execute("PRAGMA foreign_key_check").fetchone() is not None
269
+
270
+ def _is_constraint_violation(self, error: BaseException) -> bool:
271
+ return isinstance(error, sqlite3.IntegrityError)
272
+
273
+ def _drop_staging(self) -> None:
274
+ objects = self._conn.execute(
275
+ "SELECT type, name FROM sqlite_temp_master WHERE type IN ('view', 'table') ORDER BY type DESC"
276
+ ).fetchall()
277
+ for kind, name in objects:
278
+ self._conn.execute(f"DROP {kind} IF EXISTS temp.{quote(name)}")
279
+
280
+ def _attach_file(self, name: str, path: str | Path) -> None:
281
+ self._conn.execute("ATTACH DATABASE ? AS ?", (str(path), name))
282
+
283
+ def _detach_file(self, name: str) -> None:
284
+ self._conn.execute("DETACH DATABASE ?", (name,))
285
+
286
+ @staticmethod
287
+ def carry_entries(path: Path, entries: list[str]) -> None:
288
+ with closing(_existing_file(path)) as connection, connection:
289
+ connection.execute(f"CREATE TABLE {CARRIED_TABLE} (ordinal INTEGER PRIMARY KEY, entry TEXT NOT NULL)")
290
+ connection.executemany(f"INSERT INTO {CARRIED_TABLE} VALUES (?, ?)", list(enumerate(entries)))
291
+
292
+ @staticmethod
293
+ def identifiers(sql: str) -> frozenset[str]:
294
+ bare = _SINGLE_QUOTED.sub("''", _without_comments(sql))
295
+ return frozenset(name.lower() for name in _IDENTIFIER.findall(bare))
296
+
297
+ @staticmethod
298
+ def write_side_tables(path: Path, tables: Iterable[SideTable]) -> None:
299
+ with closing(sqlite3.connect(path)) as connection, connection:
300
+ for table in tables:
301
+ columns = ", ".join(f"{quote(column)} {kind}" for column, kind in table.columns)
302
+ connection.execute(f"CREATE TABLE {quote(table.name)} ({columns})")
303
+ marks = ", ".join("?" for _ in table.columns)
304
+ connection.executemany(f"INSERT INTO {quote(table.name)} VALUES ({marks})", table.rows)
305
+
306
+ @staticmethod
307
+ def take_entries(path: Path) -> list[str]:
308
+ with closing(_existing_file(path)) as connection, connection:
309
+ carried = connection.execute(
310
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (CARRIED_TABLE,)
311
+ ).fetchone()
312
+ if carried is None:
313
+ return []
314
+ entries = [entry for (entry,) in connection.execute(f"SELECT entry FROM {CARRIED_TABLE} ORDER BY ordinal")]
315
+ connection.execute(f"DROP TABLE {CARRIED_TABLE}")
316
+ return entries
317
+
318
+ @staticmethod
319
+ def drop_indexes(path: Path, names: Iterable[str]) -> None:
320
+ with closing(_existing_file(path)) as connection, connection:
321
+ held = {name for (name,) in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")}
322
+ for name in sorted(set(names) & held):
323
+ connection.execute(f"DROP INDEX {quote(name)}")
324
+
325
+ @staticmethod
326
+ def recover_copy(source: Path, destination: Path) -> None:
327
+ # Opening the file read-write is what rolls back a hot journal its dead writer left beside it.
328
+ with closing(_existing_file(source)) as live, closing(sqlite3.connect(destination)) as copy:
329
+ live.backup(copy)
330
+
331
+ @staticmethod
332
+ def stored_lock(path: Path) -> dict:
333
+ connection = sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
334
+ try:
335
+ rows = connection.execute(STORED_LOCK_QUERY).fetchall()
336
+ except sqlite3.Error as error:
337
+ raise ContractError("state_contract", "State has no compatible schema contract") from error
338
+ finally:
339
+ connection.close()
340
+ return WorldDatabase.lock_from(rows)
341
+
342
+ @staticmethod
343
+ def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
344
+ connection = sqlite3.connect(destination, isolation_level=None)
345
+ try:
346
+ connection.execute("PRAGMA foreign_keys = ON")
347
+ connection.execute("ATTACH DATABASE ? AS stored", (str(source),))
348
+ connection.execute("BEGIN IMMEDIATE")
349
+ for table in ledger:
350
+ connection.execute(f"INSERT INTO main.{quote(table)} SELECT * FROM stored.{quote(table)}")
351
+ for entity in entities:
352
+ connection.execute(
353
+ f"INSERT INTO main.{quote(entity)} (_key, payload) SELECT _key, payload FROM stored.{quote(entity)}"
354
+ )
355
+ connection.execute("COMMIT")
356
+ except sqlite3.Error as error:
357
+ raise ContractError("state_contract", f"The stored rows could not be migrated ({error})") from None
358
+ finally:
359
+ connection.close()
360
+
361
+ def _write_copy(self, destination: Path) -> None:
362
+ """Between steps the lock is let go for a waiting request; the backup runs on this connection, so a write made
363
+ then lands in the copy."""
364
+ target = sqlite3.connect(destination)
365
+ try:
366
+ with self._lock:
367
+ self._conn.backup(target, pages=SNAPSHOT_STEP_PAGES, progress=self._between_steps)
368
+ finally:
369
+ target.close()
370
+
371
+ def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
372
+ if budget is None:
373
+ self._conn.set_progress_handler(None, 0)
374
+ for limit, value in self._saved_limits.items():
375
+ self._conn.setlimit(limit, value)
376
+ self._saved_limits = {}
377
+ return
378
+ self._saved_limits[sqlite3.SQLITE_LIMIT_LENGTH] = self._conn.setlimit(
379
+ sqlite3.SQLITE_LIMIT_LENGTH, budget.value_bytes
380
+ )
381
+ self._conn.set_progress_handler(lambda: int(time.monotonic() >= deadline), PROGRESS_STEPS)
382
+
383
+ def _between_steps(self, _status: int, remaining: int, _total: int) -> None:
384
+ if not remaining:
385
+ return
386
+ self._lock.release()
387
+ try:
388
+ time.sleep(SNAPSHOT_STEP_PAUSE_S)
389
+ finally:
390
+ self._lock.acquire()
391
+
392
+ def _close_connection(self) -> None:
393
+ self._conn.close()