@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -1,12 +1,3 @@
1
- """The world database: one contract-bound store, whatever engine holds the rows.
2
-
3
- :class:`WorldDatabase` owns everything a world's state promises — primary-key
4
- identity, schema validation, atomic batches with lineage, the commit-time audit,
5
- the guard deciding who may do what — and asks its engine subclass only for the
6
- primitives: open a connection, run a statement, begin and end a transaction,
7
- list the schema objects, check relationships, copy the file.
8
- """
9
-
10
1
  from __future__ import annotations
11
2
 
12
3
  import json
@@ -17,10 +8,11 @@ import time
17
8
  import uuid
18
9
  from abc import ABC, abstractmethod
19
10
  from contextlib import contextmanager
11
+ from dataclasses import dataclass
20
12
  from pathlib import Path
21
13
  from typing import Any, Callable, Iterable, Iterator
22
14
 
23
- from ..compiler import Contract, StorageContract, field_schema, quote, scalar_types
15
+ from ..compiler import Contract, StorageContract, field_schema, quote
24
16
  from ..primary_key import PrimaryKey
25
17
  from ..validation import (
26
18
  ContractError,
@@ -40,6 +32,7 @@ from .guard import PRIVATE_TABLE_ORDER, Guard
40
32
  from .ingest import IngestSession
41
33
  from .matching import RowMatch
42
34
  from .provenance import validate_source
35
+ from .reads import ReadIndexes, Selection
43
36
  from .rows import Row, Rows
44
37
  from .runtime_rows import RuntimeRows, Statement
45
38
 
@@ -57,10 +50,6 @@ CREATE TABLE benchmark_conversation_events (scenario_id TEXT NOT NULL, user_turn
57
50
  event_type TEXT NOT NULL, actor TEXT NOT NULL, name TEXT, payload_json TEXT NOT NULL);
58
51
 
59
52
  """
60
- #: The scalar types whose stored text equals Python's ``str`` of the value, so a
61
- #: ``list_where`` filter on such a column can run in SQL and answer as :class:`RowMatch` does.
62
- #: A boolean is stored as 1/0 and read as True/False; it filters in Python.
63
- TEXT_SCALARS = {"string", "integer", "number"}
64
53
  _FICLONE = 0x40049409
65
54
 
66
55
 
@@ -75,27 +64,21 @@ def _reflink(source: Path, destination: Path) -> bool:
75
64
  return False
76
65
 
77
66
 
78
- def _sql_filterable(prop: dict) -> bool:
79
- types = scalar_types(prop)
80
- return bool(types) and types <= TEXT_SCALARS
67
+ @dataclass(frozen=True)
68
+ class SideTable:
69
+ name: str
70
+ columns: tuple[tuple[str, str], ...]
71
+ rows: tuple[tuple, ...]
81
72
 
82
73
 
83
74
  class WorldDatabase(ABC):
84
- """Use create/open as context managers; every public mutation is atomic.
75
+ """Every public mutation is atomic; ``update`` shallow-merges ``patch`` and validates the whole row."""
85
76
 
86
- Payloads are JSON values, keys are dicts of exactly the primary-key fields.
87
- Updates shallow-merge provided properties and validate the complete result.
88
- """
89
-
90
- #: The engine's name; also the key under which an ingest refusal carries the engine's own words.
91
77
  engine: str
92
78
  engine_label: str
93
- #: The storage target a contract compiles to for this engine (the lock's ``target``).
94
79
  target: str
95
- #: The file name of a snapshot this engine writes, and its media type on the wire.
96
80
  snapshot_name: str
97
81
  content_type: str
98
- #: The engine's error type: what a statement the engine itself refused raises.
99
82
  native_error: type[BaseException]
100
83
  #: Functions that reach outside the database (files, native extensions): every door but the trusted one
101
84
  #: refuses them by name.
@@ -157,7 +140,6 @@ class WorldDatabase(ABC):
157
140
 
158
141
  @classmethod
159
142
  def engine_named(cls, name: str) -> type["WorldDatabase"]:
160
- """The engine a world's manifest names (``[db] engine``)."""
161
143
  found = [engine for engine in cls.engines() if engine.engine == name]
162
144
  if not found:
163
145
  fail("target", f"{cls.requirement()}; db engine {name!r} is not one")
@@ -165,7 +147,6 @@ class WorldDatabase(ABC):
165
147
 
166
148
  @classmethod
167
149
  def for_contract(cls, contract: Contract) -> type["WorldDatabase"]:
168
- """The engine whose storage target the contract was compiled to."""
169
150
  target = contract.metadata.get("target")
170
151
  found = [engine for engine in cls.engines() if engine.target == target]
171
152
  if not found:
@@ -178,23 +159,17 @@ class WorldDatabase(ABC):
178
159
 
179
160
  @classmethod
180
161
  def open(cls, path: str | Path, contract: Contract, *, audit: bool = True) -> "WorldDatabase":
181
- """Open existing state. ``audit=False`` skips the full row audit: for a snapshot
182
- the store itself wrote and whose bytes a sidecar digest has just verified,
183
- every row was audited when it was written, so opening is then a matter of
184
- milliseconds however many rows the file holds. The contract check always runs."""
162
+ """``audit=False`` only for a file the store wrote and a digest verified; the contract check always runs."""
185
163
  return cls.for_contract(contract)(Path(path), contract, audit=audit)
186
164
 
187
165
  @classmethod
188
166
  def open_copy(cls, source: str | Path, contract: Contract, destination: str | Path) -> "WorldDatabase":
189
- """Open a private copy of a snapshot the store wrote and a digest verified."""
167
+ """Only for a snapshot the store wrote and a digest verified: the copy opens unaudited."""
190
168
  cls.clone_file(Path(source), Path(destination))
191
169
  return cls.open(destination, contract, audit=False)
192
170
 
193
171
  @staticmethod
194
172
  def clone_file(source: Path, destination: Path) -> None:
195
- """Copy a database file cheaply: a reflink (``FICLONE``) shares the source's
196
- blocks copy-on-write, so a session costs the same whatever the snapshot's size;
197
- a file system or platform without reflink falls back to a plain copy."""
198
173
  if not stat.S_ISREG(source.stat().st_mode):
199
174
  raise shutil.SpecialFileError(f"{source} is not a regular file")
200
175
  destination.parent.mkdir(parents=True, exist_ok=True)
@@ -205,52 +180,63 @@ class WorldDatabase(ABC):
205
180
 
206
181
  @classmethod
207
182
  @abstractmethod
208
- def ddl(cls, entities: dict) -> str:
209
- """The storage DDL a contract's compiled entities become (``db/schema.sql``)."""
183
+ def ddl(cls, entities: dict) -> str: ...
210
184
 
211
185
  @staticmethod
212
186
  @abstractmethod
213
- def split_statements(script: str) -> list[str]:
214
- """A script as its complete statements, comment-only text dropped."""
187
+ def split_statements(script: str) -> list[str]: ...
215
188
 
216
189
  @staticmethod
217
190
  @abstractmethod
218
- def missing_table(error: BaseException) -> str | None:
219
- """The table an engine error says does not exist, or ``None``."""
191
+ def missing_table(error: BaseException) -> str | None: ...
220
192
 
221
193
  @staticmethod
222
194
  @abstractmethod
223
- def exceeds_limit(error: BaseException) -> bool:
224
- """Whether an engine error says a value or a row passed the engine's size limits."""
195
+ def exceeds_limit(error: BaseException) -> bool: ...
225
196
 
226
197
  @staticmethod
227
198
  @abstractmethod
228
- def carry_entries(path: Path, entries: list[str]) -> None:
229
- """Write ``entries`` into a closed database file beside its rows, in a table the store never opens."""
199
+ def carry_entries(path: Path, entries: list[str]) -> None: ...
230
200
 
231
201
  @staticmethod
232
202
  @abstractmethod
233
- def stored_lock(path: Path) -> dict:
234
- """The lock a database file records its rows under, read without opening it as a store."""
203
+ def stored_lock(path: Path) -> dict: ...
235
204
 
236
205
  @staticmethod
237
206
  @abstractmethod
238
207
  def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
239
- """Every row of ``source`` copied into the new database at ``destination`` in one transaction:
240
- the ``ledger`` tables whole, each entity's keys and payloads, relationships checked at commit."""
208
+ """In one transaction, relationships checked at commit."""
209
+
210
+ @staticmethod
211
+ @abstractmethod
212
+ def take_entries(path: Path) -> list[str]: ...
213
+
214
+ @staticmethod
215
+ @abstractmethod
216
+ def write_side_tables(path: Path, tables: Iterable[SideTable]) -> None: ...
241
217
 
242
218
  @staticmethod
243
219
  @abstractmethod
244
- def take_entries(path: Path) -> list[str]:
245
- """The entries :meth:`carry_entries` wrote into ``path``, in order, dropped from it; empty when none."""
220
+ def identifiers(sql: str) -> frozenset[str]:
221
+ """The names a statement uses outside its comments and string literals, lowercased."""
222
+
223
+ @staticmethod
224
+ @abstractmethod
225
+ def drop_indexes(path: Path, names: Iterable[str]) -> None: ...
226
+
227
+ @staticmethod
228
+ @abstractmethod
229
+ def recover_copy(source: Path, destination: Path) -> None: ...
230
+
231
+ @classmethod
232
+ @abstractmethod
233
+ def match_text(cls, column: str) -> str: ...
246
234
 
247
235
  @abstractmethod
248
236
  def _open_connection(self) -> None: ...
249
237
 
250
238
  @abstractmethod
251
- def _enforce_strict_dialect(self) -> None:
252
- """Refuse what would let a verifier score silently wrong (a missing column
253
- read as a literal) and turn on relationship enforcement."""
239
+ def _enforce_strict_dialect(self) -> None: ...
254
240
 
255
241
  @abstractmethod
256
242
  def _execute(self, sql: str, params: Any = ()) -> Any: ...
@@ -262,16 +248,13 @@ class WorldDatabase(ABC):
262
248
  def _execute_script(self, script: str) -> None: ...
263
249
 
264
250
  @abstractmethod
265
- def _schema_objects(self) -> dict[str, str]:
266
- """Every schema object the file holds, by name, with its DDL."""
251
+ def _schema_objects(self) -> dict[str, str]: ...
267
252
 
268
253
  @abstractmethod
269
- def _expected_schema_objects(self) -> dict[str, str]:
270
- """The schema objects a fresh file built from :meth:`_schema_script` holds."""
254
+ def _objects_of(self, script: str) -> dict[str, str]: ...
271
255
 
272
256
  @abstractmethod
273
- def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
274
- """Make ``gateway_key(entity, payload)`` callable from SQL."""
257
+ def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None: ...
275
258
 
276
259
  @abstractmethod
277
260
  def _install_guard(self) -> None:
@@ -297,40 +280,35 @@ class WorldDatabase(ABC):
297
280
  def _rollback_to(self, name: str) -> None: ...
298
281
 
299
282
  @abstractmethod
300
- def _page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
301
- """``LIMIT``/``OFFSET`` text and its parameters; ``limit=None`` is every row."""
283
+ def page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]: ...
302
284
 
303
285
  @abstractmethod
304
- def _relationships_broken(self) -> bool:
305
- """Whether any persisted row references a row that does not exist."""
286
+ def _relationships_broken(self) -> bool: ...
306
287
 
307
288
  @abstractmethod
308
- def _is_constraint_violation(self, error: BaseException) -> bool:
309
- """Whether a row write failed on a declared key or constraint."""
289
+ def _is_constraint_violation(self, error: BaseException) -> bool: ...
310
290
 
311
291
  @abstractmethod
312
- def _drop_staging(self) -> None:
313
- """Drop the TEMP tables and views an ingest session left behind."""
292
+ def _drop_staging(self) -> None: ...
314
293
 
315
294
  @abstractmethod
316
295
  def _attach_file(self, name: str, path: str | Path) -> None: ...
317
296
 
318
297
  @abstractmethod
319
- def _write_copy(self, destination: Path) -> None:
320
- """A consistent copy of the committed state at ``destination``."""
298
+ def _detach_file(self, name: str) -> None: ...
299
+
300
+ @abstractmethod
301
+ def _write_copy(self, destination: Path) -> None: ...
321
302
 
322
303
  @abstractmethod
323
304
  def _close_connection(self) -> None: ...
324
305
 
325
306
  @abstractmethod
326
- def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
327
- """Interrupt any statement still running at ``deadline`` (``time.monotonic``) and refuse a value
328
- or a row wider than ``budget`` allows; ``None`` restores what the connection had before."""
307
+ def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None: ...
329
308
 
330
309
  @property
331
310
  @abstractmethod
332
- def insertion_order(self) -> str:
333
- """An expression ordering a bookkeeping table's rows as they were inserted."""
311
+ def insertion_order(self) -> str: ...
334
312
 
335
313
  def _schema_script(self) -> str:
336
314
  return self.contract.artifacts["db/schema.sql"].decode() + BOOKKEEPING_DDL
@@ -349,9 +327,12 @@ class WorldDatabase(ABC):
349
327
  )
350
328
  fail("state_contract", message, changed=changed)
351
329
  # The live DDL is compared with freshly compiled DDL, not a stored hash a tampered file could carry.
352
- expected = self._expected_schema_objects()
353
330
  actual = self._schema_objects()
354
- if any(actual.get(k) != v for k, v in expected.items()) or set(actual) - set(expected):
331
+ reads = ReadIndexes.of(self, ReadIndexes.parse(actual)).by_name()
332
+ built = self._objects_of(self._schema_script() + "".join(f"{sql};\n" for sql in reads.values()))
333
+ expected = {name: sql for name, sql in built.items() if name not in reads}
334
+ extra = {name: sql for name, sql in actual.items() if name not in expected}
335
+ if any(actual.get(k) != v for k, v in expected.items()) or any(built.get(n) != v for n, v in extra.items()):
355
336
  fail(
356
337
  "state_contract",
357
338
  "State DDL differs from the compiled storage contract",
@@ -364,7 +345,6 @@ class WorldDatabase(ABC):
364
345
 
365
346
  @staticmethod
366
347
  def lock_from(rows: Iterable[Any]) -> dict:
367
- """The one lock a database's ``_gateway_contract`` rows record."""
368
348
  try:
369
349
  (lock,) = [json.loads(text) for (text,) in rows]
370
350
  except (TypeError, ValueError):
@@ -375,11 +355,7 @@ class WorldDatabase(ABC):
375
355
  return self._guard.allows(access)
376
356
 
377
357
  def _sql_key(self, entity: Any, payload: Any) -> str | None:
378
- """``gateway_key(entity, payload)``: the encoded ``_key`` for a payload.
379
-
380
- Returns NULL on an invalid entity or payload so the NOT NULL constraint
381
- rejects the row; the reason is attached to the resulting error.
382
- """
358
+ """NULL on an invalid entity or payload, so NOT NULL rejects the row; the reason is attached to its error."""
383
359
  try:
384
360
  value = payload
385
361
  if isinstance(payload, (str, bytes)):
@@ -480,25 +456,12 @@ class WorldDatabase(ABC):
480
456
  return [json.loads(row[0]) for row in rows]
481
457
 
482
458
  def list_where(
483
- self,
484
- entity: str,
485
- where: dict[str, list],
486
- limit: int | None = None,
487
- offset: int = 0,
459
+ self, entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0
488
460
  ) -> tuple[list[dict], int]:
489
- """A page of the rows of ``entity`` whose every ``where`` field holds one of
490
- the listed values, and how many rows match in all: ``(rows, total)``.
491
-
492
- Equality is the routing rule of :class:`RowMatch`: the stored value as
493
- text, case-insensitively; a list of wanted values matches when any does
494
- (IN); a list field matches when any element does. A field projected to a
495
- column (string, integer, number) filters in SQL through the read-only
496
- guard, so a large world pages without loading; a field that is not
497
- (an array, an object, a boolean) filters the listed rows in Python with the
498
- same rule, so both paths answer alike. A missing field never matches."""
461
+ """A page of ``entity``'s rows whose every ``where`` field holds one of its values, and the total match count.
462
+ Values compare as text, case-insensitively; a list field matches when any element does, a missing one never."""
499
463
  with self._lock:
500
- spec = self._entity(entity)
501
- properties = spec["schema"].get("properties", {})
464
+ properties = self._entity(entity)["schema"].get("properties", {})
502
465
  unknown = sorted(set(where) - set(properties))
503
466
  if unknown:
504
467
  fail(
@@ -510,81 +473,75 @@ class WorldDatabase(ABC):
510
473
  if offset < 0:
511
474
  fail("input", "offset must be zero or a positive integer")
512
475
  wanted = {field: [str(value) for value in values] for field, values in where.items()}
513
- if any(not values for values in wanted.values()):
514
- return [], 0
515
- if all(_sql_filterable(properties[field]) for field in wanted):
516
- return self._list_where_sql(entity, wanted, limit, offset)
517
- matched = RowMatch.on_fields(wanted).select(self.list(entity))
518
- end = None if limit is None else offset + limit
519
- return matched[offset:end], len(matched)
520
-
521
- def _list_where_sql(
522
- self, entity: str, wanted: dict[str, list[str]], limit: int | None, offset: int
523
- ) -> tuple[list[dict], int]:
524
- clauses = []
525
- params: list = []
526
- for field, values in wanted.items():
527
- clauses.append(f"lower(CAST({quote(field)} AS TEXT)) IN ({', '.join('?' for _ in values)})")
528
- params.extend(value.lower() for value in values)
529
- condition = f" WHERE {' AND '.join(clauses)}" if clauses else ""
530
- table = quote(entity)
531
- total = self.query(f"SELECT COUNT(*) FROM {table}{condition}", tuple(params))[0][0]
532
- clause, paging = self._page_clause(limit, offset)
533
- page = self.query(f"SELECT payload FROM {table}{condition} ORDER BY _key{clause}", (*params, *paging))
534
- return [json.loads(row[0]) for row in page], total
535
-
536
- def attach(self, name: str, path: str | Path, entities: Iterable[str]) -> None:
537
- """Read another store's file as ``<name>.<table>`` from :meth:`query` and
538
- the verifiers that run through it: a cross-world check joins its own
539
- rows to a dependency world's (``SELECT ... FROM slack.slack_messages``).
540
- Only the ``entities`` named are readable and nothing is writable, so a
541
- tool of this world cannot reach into another's store."""
476
+ return self.select(entity, RowMatch.on_fields(wanted), limit, offset)
477
+
478
+ def select(self, entity: str, match: RowMatch, limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]:
479
+ """Equals ``match.select(self.list(entity))`` sliced to a page, and its length, read by the engine."""
480
+ with self._lock:
481
+ self._entity(entity)
482
+ return Selection(self, entity, match).page(limit, offset)
483
+
484
+ def index_reads(self, extra: Iterable[tuple[str, tuple[str, ...]]] = ()) -> None:
485
+ with self._lock, self._trusted():
486
+ held = self._schema_objects()
487
+ for name, statement in ReadIndexes.of(self, extra).by_name().items():
488
+ if name not in held:
489
+ self._execute(statement)
490
+
491
+ def attach(self, name: str, path: str | Path, entities: Iterable[str], *, bare: bool = False) -> None:
492
+ """Only ``entities`` are readable and nothing is writable: a tool cannot reach into another world's store."""
542
493
  if self._guard.is_attached(name):
543
494
  fail("attach", f"'{name}' is attached already")
544
495
  with self._lock, self._trusted():
545
496
  self._attach_file(name, path)
546
- self._guard.attach(name, frozenset(entities))
497
+ self._guard.attach(name, frozenset(entities), bare=bare)
498
+
499
+ def detach(self, name: str) -> None:
500
+ if not self._guard.is_attached(name):
501
+ return
502
+ self._guard.detach(name)
503
+ with self._lock, self._trusted():
504
+ self._detach_file(name)
505
+
506
+ def reads_of(self, name: str) -> int:
507
+ return self._guard.reads(name)
508
+
509
+ def columns_read_of(self, name: str) -> frozenset[str]:
510
+ return self._guard.columns(name)
547
511
 
548
512
  @property
549
513
  def attached(self) -> dict[str, frozenset[str]]:
550
- """The attached stores by name, each with the entities readable there."""
551
514
  return dict(self._guard.attached)
552
515
 
553
516
  def read_connection(self) -> ReadConnection:
554
- """What a tool handler receives as ``conn``: reads through the read-only guard, no writes."""
517
+ """A tool handler's ``conn``: reads through the read-only guard, no writes."""
555
518
  return ReadConnection(self)
556
519
 
557
520
  def key_of(self, entity: str, row: dict) -> str:
558
- """The encoded identity of ``row`` in ``entity``: its primary-key fields, validated."""
559
521
  with self._lock:
560
522
  return self._key(entity, row)
561
523
 
562
524
  def keys(self, entity: str) -> set[str]:
563
- """The encoded identity of every row ``entity`` holds."""
564
525
  with self._lock:
565
526
  self._entity(entity)
566
527
  with self._trusted():
567
528
  return {row[0] for row in self._execute(f"SELECT _key FROM {quote(entity)}")}
568
529
 
569
530
  def set_task_context(self, scenario_id: str) -> None:
570
- """Name the active task in ``benchmark_runtime_context``, so a verifier keys off its scenario."""
571
531
  self._write_runtime(RuntimeRows.task_context(scenario_id))
572
532
 
573
533
  def set_run_parameters(self, parameters: dict[str, str]) -> None:
574
- """Replace ``benchmark_runtime_parameters`` with the run's condition bindings."""
575
534
  self._write_runtime(RuntimeRows.run_parameters(parameters))
576
535
 
577
536
  def record_conversation_event(
578
537
  self, scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
579
538
  ) -> None:
580
- """Append one turn event to ``benchmark_conversation_events``, read only by a verifier."""
581
539
  self._write_runtime(
582
540
  RuntimeRows.conversation_event(scenario_id, user_turn_index, event_type, actor, name, payload_json)
583
541
  )
584
542
 
585
543
  def _write_runtime(self, statements: list[Statement]) -> None:
586
- """Store-authored writes to the runtime tables, all or none. They are not
587
- entities, so no audit runs; inside an open transaction they ride a savepoint."""
544
+ """Runtime rows are not entities, so no audit runs."""
588
545
  with self._lock:
589
546
  if self._guard.ingesting:
590
547
  fail("transaction", "Runtime rows are written outside an ingest session")
@@ -601,8 +558,7 @@ class WorldDatabase(ABC):
601
558
  raise
602
559
 
603
560
  def query(self, sql: str, params: tuple = (), *, private: bool = False, limit: int | None = None) -> Rows:
604
- """Read-only SQL over domain tables, enforced by the guard; rows are :class:`Row`, never the
605
- engine's own. With ``limit``, at most that many rows are read from the engine."""
561
+ """Read-only SQL over domain tables, enforced by the guard."""
606
562
  with self._lock, self._guard.reading(private):
607
563
  cursor = None
608
564
  try:
@@ -640,9 +596,6 @@ class WorldDatabase(ABC):
640
596
 
641
597
  @contextmanager
642
598
  def bounded(self, budget: QueryBudget) -> Iterator["WorldDatabase"]:
643
- """Every :meth:`query` inside runs within ``budget``: past its time the statement is stopped
644
- (``time``), past its answer bytes the read is refused (``size``), and the engine refuses a
645
- value larger than it allows (``size``). One budget at a time per connection."""
646
599
  with self._lock:
647
600
  if self._budget is not None:
648
601
  fail("budget", "A query budget is already in force on this connection")
@@ -657,9 +610,7 @@ class WorldDatabase(ABC):
657
610
  self._set_budget(None, None)
658
611
 
659
612
  def public_copy(self, destination: str | Path) -> "WorldDatabase":
660
- """A copy of the committed state at ``destination`` holding no bookkeeping or private runtime
661
- rows (contract, batches, lineage, task context, conversation events): what a caller's own SQL
662
- may open, so a read the engine's guard misses finds nothing private to return."""
613
+ """Holds no bookkeeping or private runtime rows, so a read the guard misses finds nothing private to return."""
663
614
  with self._lock:
664
615
  if self._depth:
665
616
  fail("transaction", "Cannot copy uncommitted state")
@@ -711,7 +662,6 @@ class WorldDatabase(ABC):
711
662
  return key
712
663
 
713
664
  def _merged(self, name: str, entity: dict, key: str, mutation: dict) -> dict | None:
714
- """The row an update leaves (validated, key unchanged); ``None`` for a delete."""
715
665
  current = self.get(name, mutation["key"])
716
666
  if current is None:
717
667
  fail(
@@ -904,17 +854,8 @@ class WorldDatabase(ABC):
904
854
  batch_id: str | None = None,
905
855
  limit: int = 50,
906
856
  ) -> Iterator[IngestConnection]:
907
- """Schema-gated raw SQL: write freely, commit only if every row conforms.
908
-
909
- The session yields a connection that may INSERT/UPDATE/DELETE domain
910
- tables and create TEMP staging objects. ``gateway_key(entity, payload)``
911
- computes ``_key`` in SQL. On exit the store validates all persisted
912
- state; any violation (up to ``limit`` are reported together) rolls the
913
- whole session back. Committed changes are recorded as one batch with a
914
- lineage row per touched key, so raw ingestion keeps provenance. Known gap:
915
- a TEMP object named like a main table still shadows it (open bug
916
- ``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, to be fixed by #474).
917
- """
857
+ """Commits only if every row conforms (up to ``limit`` violations reported together). Known gap (#474): a TEMP
858
+ object named like a main table still shadows it."""
918
859
  session = IngestSession(self, source=source, mapping=mapping, batch_id=batch_id, limit=limit)
919
860
  yield from session.run()
920
861
 
@@ -940,7 +881,6 @@ class WorldDatabase(ABC):
940
881
  ]
941
882
 
942
883
  def audit(self) -> None:
943
- """Validate real persisted payloads, key identity and strict references."""
944
884
  with self._lock:
945
885
  violations = self._audit.violations(limit=1)
946
886
  if violations:
@@ -955,10 +895,8 @@ class WorldDatabase(ABC):
955
895
  }
956
896
 
957
897
  def snapshot(self, destination: str | Path) -> Path:
958
- """A consistent copy of the committed state, written by the engine's :meth:`_write_copy`.
959
- It is not audited again: every transaction was audited as it committed
960
- (:meth:`_audit_touched`). The engine may let the lock go between copy steps;
961
- :meth:`close` waits for a copy in progress."""
898
+ """Drops the :class:`ReadIndexes`, so its bytes do not depend on where it was taken. The engine may let the
899
+ lock go between copy steps; :meth:`close` waits for a copy in progress."""
962
900
  dest = Path(destination)
963
901
  with self._lock:
964
902
  if self._depth:
@@ -971,6 +909,9 @@ class WorldDatabase(ABC):
971
909
  self._copies += 1
972
910
  try:
973
911
  self._write_copy(dest)
912
+ with self._lock:
913
+ held = self._schema_objects()
914
+ self.drop_indexes(dest, ReadIndexes.named(held))
974
915
  except BaseException:
975
916
  dest.unlink(missing_ok=True)
976
917
  raise