@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -18,8 +18,9 @@ way that kind seeds:
18
18
  are generated or per-task; the import refuses and names what to edit.
19
19
 
20
20
  The same row planning seeds a *live* session: :func:`seed_batch` builds the
21
- batch a running store applies atomically, and :func:`seed_sqlite` executes
22
- the inserts against a task's live sqlite.
21
+ batch a running store applies atomically, and
22
+ :func:`~gatewaysdk.environment.legacy_sqlite.seed_sqlite` executes the inserts
23
+ against a mock-tools task's live database.
23
24
  """
24
25
 
25
26
  from __future__ import annotations
@@ -28,22 +29,22 @@ import hashlib
28
29
  import json
29
30
  import re
30
31
  import shutil
31
- import sqlite3
32
32
  import tempfile
33
33
  import uuid
34
34
  from contextlib import contextmanager
35
35
  from pathlib import Path
36
36
  from typing import Any, Iterable, Iterator
37
37
 
38
+ from ..legacy_sqlite import MODES, LegacySQLiteBacking, schema_columns, seed_statements
38
39
  from ..world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, SCHEMA_MANIFEST_KIND, World
39
40
  from .cast import EntityCasters
40
41
  from .compiler import bundle_mode, check_bundle, compile_bundle
42
+ from .database import RowMatch, WorldDatabase
43
+ from .primary_key import PrimaryKey
41
44
  from .proof import SCRATCH_IGNORE, RowProof
42
- from .store import SQLiteStore
43
45
  from .tree_import import refuse_unpublishable_journal
44
- from .validation import ContractError, digest, encoded, fail, journal_paths, read_json, validate
46
+ from .validation import ContractError, digest, encoded, fail, journal_paths, read_json
45
47
 
46
- MODES = ("append", "replace")
47
48
  MERGES = ("upsert", "fill")
48
49
  #: The inline lane's caps (the platform's seed and import calls share them): a local
49
50
  #: directory import above either is refused; larger data is a platform batch.
@@ -58,9 +59,6 @@ MAPPING = {"id": "data-import", "version": "1"}
58
59
  Rows = dict[str, list[dict]]
59
60
 
60
61
 
61
- # ---------------------------------------------------------------- rows files
62
-
63
-
64
62
  def parse_rows(
65
63
  paths: Iterable[str | Path],
66
64
  *,
@@ -94,9 +92,6 @@ def parse_rows(
94
92
  return rows
95
93
 
96
94
 
97
- # ---------------------------------------------------------------- world kinds
98
-
99
-
100
95
  def world_kind(root: str | Path) -> str:
101
96
  """The kind the worlds paths report (``World.from_dir(...).reported_kind``):
102
97
  ``schema``, ``mock-tools``, ``api``, ``harbor``, ``verifiers`` or
@@ -131,9 +126,6 @@ def _refuse_kind(kind: str, root: Path) -> None:
131
126
  )
132
127
 
133
128
 
134
- # ---------------------------------------------------------------- import
135
-
136
-
137
129
  def import_rows(
138
130
  root: str | Path,
139
131
  rows: Rows,
@@ -201,7 +193,7 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
201
193
  the row's index in the file, carrying the full ``refusals`` list — each
202
194
  naming its entity, index, primary key and field — the shape the batch lane
203
195
  reports."""
204
- from .batch import shape_refusal
196
+ from .batch import raise_shape_refusal, shape_refusal
205
197
 
206
198
  contract = compile_bundle(root, snapshots=False)
207
199
  refusals = [
@@ -214,22 +206,10 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
214
206
  return
215
207
  first = refusals[0]
216
208
  entity, index = first["entity"], first["index"]
217
- row = rows[entity][index]
218
209
  total = sum(len(listed) for listed in rows.values())
219
210
  summary = f"{len(refusals)} of {total} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
220
211
  context = {"entity": entity, "index": index, "refused": len(refusals), "rows": total, "refusals": refusals}
221
- if entity not in contract.entities:
222
- fail("entity", summary + first["detail"], f"/{entity}", **context)
223
- if not isinstance(row, dict):
224
- fail("rows", summary + first["detail"], first["pointer"], **context)
225
- spec = contract.entities[entity]
226
- if any(name not in row for name in spec["primary_key"]):
227
- fail("primary_key", summary + first["detail"], first["pointer"], **context)
228
- try:
229
- validate(spec["schema"], row, location=f"{spec['location']}/schema")
230
- except ContractError as error:
231
- raise error.contextualize(message=summary + error.detail["message"], **context)
232
- fail("rows", summary + first["detail"], first["pointer"], **context)
212
+ raise_shape_refusal(contract, first, rows[entity][index], summary, context, f"/{entity}")
233
213
 
234
214
 
235
215
  def _merged(root: Path, rows: Rows, merge: str) -> Rows:
@@ -266,16 +246,9 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
266
246
  the next journal instead: a replace deletes rows those journals hold, and
267
247
  an append upserts against them, rather than being overwritten on replay.
268
248
  """
269
- from .snapshot import pin_path
249
+ from .snapshot import refuse_pinned_rows
270
250
 
271
- if pin_path(root).is_file():
272
- fail(
273
- "snapshot_pinned",
274
- "This world's data is a hosted snapshot (db/snapshot.json), which its data files "
275
- "cannot reproduce: import rows through the platform (`gateway worlds data import "
276
- "<slug>` or import_world_data) instead of into the checkout",
277
- "/db/snapshot.json",
278
- )
251
+ refuse_pinned_rows(root)
279
252
  contract = compile_bundle(root)
280
253
  journals = journal_paths(root)
281
254
  initial_path = root / "data/initial.json"
@@ -287,8 +260,7 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
287
260
  if not fresh:
288
261
  return {"kind": KIND_SCHEMA, "batch": prior_id, "entity_counts": entity_counts(root), "changed": False}
289
262
  if mode == "replace" and not refreshed:
290
- # Folded into initial.json, a replace is written compactly: the named
291
- # entities' old mutations are dropped and the rows inserted afresh.
263
+ # Folded into initial.json: the named entities' old mutations are dropped and the rows inserted afresh.
292
264
  prior = [m for m in prior if m.get("entity") not in rows]
293
265
  fresh = _plan(contract, rows, mode, _held(contract, prior))
294
266
  match = re.search(r"\.v(\d+)$", prior_id)
@@ -323,15 +295,15 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
323
295
  (staged / written).parent.mkdir(parents=True, exist_ok=True)
324
296
  (staged / written).write_bytes(payload)
325
297
  # Building the staged snapshot is the proof: every row goes through the store.
326
- counts = build_default(staged, compile_bundle(staged, snapshots=False))["entityCounts"]
298
+ contract = compile_bundle(staged, snapshots=False)
299
+ counts = build_default(staged, contract)["entityCounts"]
327
300
  (root / written).parent.mkdir(parents=True, exist_ok=True)
328
301
  (root / written).write_bytes(payload)
329
- # The proven snapshot moves into the world; compile finds it current and only
330
- # rebuilds the scenario snapshots on top of it.
331
- target = default_snapshot(root)
302
+ # The proven snapshot moves in; compile then finds it current and only rebuilds the scenarios.
303
+ target = default_snapshot(root, contract)
332
304
  target.parent.mkdir(parents=True, exist_ok=True)
333
- shutil.move(str(default_snapshot(staged)), str(target))
334
- shutil.move(str(sidecar_path(default_snapshot(staged))), str(sidecar_path(target)))
305
+ shutil.move(str(default_snapshot(staged, contract)), str(target))
306
+ shutil.move(str(sidecar_path(default_snapshot(staged, contract))), str(sidecar_path(target)))
335
307
  compile_bundle(root)
336
308
  check_bundle(root)
337
309
  return {"kind": KIND_SCHEMA, "batch": batch["id"], "entity_counts": counts, "changed": True}
@@ -363,10 +335,7 @@ def _held(contract: Any, mutations: list[dict]) -> dict[tuple[str, tuple], dict]
363
335
 
364
336
 
365
337
  def _key(contract: Any, entity: str, row: dict) -> tuple | None:
366
- fields = contract.entities.get(entity, {}).get("primary_key", [])
367
- if not fields or any(field not in row for field in fields):
368
- return None
369
- return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
338
+ return PrimaryKey.in_contract(contract, entity).identity(row)
370
339
 
371
340
 
372
341
  def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], dict]) -> list[dict]:
@@ -389,12 +358,11 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
389
358
  }
390
359
  for (entity, key), value in list(held.items()):
391
360
  if entity in rows and (entity, key) not in incoming:
392
- fields = contract.entities[entity]["primary_key"]
393
361
  mutations.append(
394
362
  {
395
363
  "op": "delete",
396
364
  "entity": entity,
397
- "key": {field: value[field] for field in fields},
365
+ "key": PrimaryKey.in_contract(contract, entity).key(value),
398
366
  "source": {"record": f"data-import:{entity}:replace"},
399
367
  }
400
368
  )
@@ -404,13 +372,13 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
404
372
  fail(
405
373
  "entity", f"Unknown entity {entity}; the world has {', '.join(sorted(contract.entities))}", f"/{entity}"
406
374
  )
407
- fields = contract.entities[entity]["primary_key"]
375
+ primary = PrimaryKey.in_contract(contract, entity)
408
376
  for index, row in enumerate(listed):
409
377
  if not isinstance(row, dict):
410
378
  fail("rows", "A row must be an object", f"/{entity}/{index}")
411
- missing = [field for field in fields if field not in row]
412
- if missing:
413
- fail("primary_key", f"Row is missing primary key field {missing[0]!r}", f"/{entity}/{index}")
379
+ missing = primary.missing(row)
380
+ if missing is not None:
381
+ fail("primary_key", f"Row is missing primary key field {missing!r}", f"/{entity}/{index}")
414
382
  key = _key(contract, entity, row)
415
383
  assert key is not None
416
384
  existing = held.get((entity, key))
@@ -424,7 +392,7 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
424
392
  "source": {"record": f"data-import:{entity}:{identity}"},
425
393
  }
426
394
  if existing is not None:
427
- mutation["key"] = {field: row[field] for field in fields}
395
+ mutation["key"] = primary.key(row)
428
396
  mutations.append(mutation)
429
397
  held[(entity, key)] = row
430
398
  return mutations
@@ -445,22 +413,18 @@ def seed_batch(store: Any, contract: Any, rows: Rows, mode: str = "append", *, b
445
413
  if entity not in contract.entities:
446
414
  continue
447
415
  if mode == "replace":
448
- # Every held row of a replaced entity goes; the plan needs them all.
449
416
  for value in store.list(entity):
450
417
  key = _key(contract, entity, value)
451
418
  if key is not None:
452
419
  held[(entity, key)] = value
453
420
  continue
454
- # An append only needs the rows at the keys it names, however large the entity.
455
- fields = contract.entities[entity]["primary_key"]
421
+ primary = PrimaryKey.in_contract(contract, entity)
456
422
  for row in listed:
457
- if not isinstance(row, dict) or any(field not in row for field in fields):
458
- continue
459
- key = _key(contract, entity, row)
423
+ key = _key(contract, entity, row) if isinstance(row, dict) else None
460
424
  if key is None or (entity, key) in held:
461
425
  continue
462
426
  try:
463
- value = store.get(entity, {field: row[field] for field in fields})
427
+ value = store.get(entity, primary.key(row))
464
428
  except ContractError:
465
429
  continue # a key the contract refuses is refused by the plan, with the field named
466
430
  if value is not None:
@@ -475,20 +439,11 @@ def seed_batch(store: Any, contract: Any, rows: Rows, mode: str = "append", *, b
475
439
  }
476
440
 
477
441
 
478
- # ---------------------------------------------------------------- mock-tools
479
-
480
-
481
442
  def _import_mock_tools(root: Path, rows: Rows, mode: str) -> dict:
482
443
  schema = root / "db/schema.sql"
483
444
  if not schema.is_file():
484
445
  fail("schema", "mock-tools world has no db/schema.sql to read tables from", "db/schema.sql")
485
- with sqlite3.connect(":memory:") as probe:
486
- try:
487
- probe.executescript(schema.read_text(encoding="utf-8"))
488
- except sqlite3.Error as error:
489
- fail("schema", f"db/schema.sql does not load: {error}", "db/schema.sql")
490
- columns = sqlite_columns(probe)
491
- statements = sqlite_statements(columns, rows, mode)
446
+ statements = seed_statements(schema_columns(schema), rows, mode)
492
447
  if not statements:
493
448
  return {"kind": KIND_MOCK_TOOLS, "entity_counts": entity_counts(root), "changed": False}
494
449
  seed = root / "db/seed.sql"
@@ -504,117 +459,17 @@ def _import_mock_tools(root: Path, rows: Rows, mode: str) -> dict:
504
459
  (staged / "db/seed.sql").write_text(appended, encoding="utf-8")
505
460
  database = Path(scratch) / "world.db"
506
461
  materialize_template(staged, database, require_sql=True)
507
- counts = _sqlite_counts(database)
462
+ counts = _legacy_counts(database)
508
463
  seed.write_text(appended, encoding="utf-8")
509
464
  return {"kind": KIND_MOCK_TOOLS, "entity_counts": counts, "changed": True, "statements": len(statements)}
510
465
 
511
466
 
512
- def sqlite_columns(conn: sqlite3.Connection) -> dict[str, list[str]]:
513
- """Every user table's columns, in declaration order."""
514
- tables = [
515
- row[0]
516
- for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")
517
- ]
518
- return {table: [row[1] for row in conn.execute(f"PRAGMA table_info({sql_ident(table)})")] for table in tables}
519
-
520
-
521
- def sql_ident(name: str) -> str:
522
- """A SQLite identifier, quoted: a table or column named by a schema is a
523
- name, never SQL, whatever characters it carries."""
524
- return '"' + name.replace('"', '""') + '"'
525
-
526
-
527
- def _check_sqlite_rows(columns: dict[str, list[str]], rows: Rows) -> None:
528
- for table, listed in rows.items():
529
- if table not in columns:
530
- fail(
531
- "table",
532
- f"No table {table} in db/schema.sql; tables: {', '.join(sorted(columns)) or 'none'}",
533
- f"/{table}",
534
- )
535
- for index, row in enumerate(listed):
536
- if not isinstance(row, dict):
537
- fail("rows", "A row must be an object", f"/{table}/{index}")
538
- for column, value in row.items():
539
- if column not in columns[table]:
540
- fail(
541
- "column",
542
- f"Column {column} is not in table {table}; columns: {', '.join(columns[table])}",
543
- f"/{table}/{index}/{column}",
544
- )
545
- if not isinstance(value, (str, int, float, bool, type(None))):
546
- fail(
547
- "value",
548
- f"Column {column} of {table} takes a scalar; encode nested values as text yourself",
549
- f"/{table}/{index}/{column}",
550
- )
551
-
552
-
553
- def sqlite_statements(columns: dict[str, list[str]], rows: Rows, mode: str = "append") -> list[str]:
554
- """SQL text that seeds ``rows`` into tables with these ``columns``.
555
-
556
- ``append`` writes ``INSERT OR REPLACE`` (an upsert by the table's own
557
- primary key); ``replace`` prefixes ``DELETE FROM`` for each named table and
558
- writes plain inserts.
559
- """
560
- if mode not in MODES:
561
- fail("mode", f"mode must be one of {', '.join(MODES)}", "/mode")
562
- _check_sqlite_rows(columns, rows)
563
- statements: list[str] = []
564
- verb = "INSERT" if mode == "replace" else "INSERT OR REPLACE"
565
- for table, listed in rows.items():
566
- if mode == "replace":
567
- statements.append(f"DELETE FROM {sql_ident(table)};")
568
- for row in listed:
569
- if not row:
570
- statements.append(f"{verb} INTO {sql_ident(table)} DEFAULT VALUES;")
571
- continue
572
- names = ", ".join(sql_ident(column) for column in row)
573
- values = ", ".join(_sql_literal(value) for value in row.values())
574
- statements.append(f"{verb} INTO {sql_ident(table)} ({names}) VALUES ({values});")
575
- return statements
576
-
577
-
578
- def _sql_literal(value: Any) -> str:
579
- if value is None:
580
- return "NULL"
581
- if isinstance(value, bool):
582
- return "1" if value else "0"
583
- if isinstance(value, (int, float)):
584
- return repr(value)
585
- return "'" + str(value).replace("'", "''") + "'"
586
-
587
-
588
- def seed_sqlite(conn: sqlite3.Connection, rows: Rows, mode: str = "append") -> dict[str, int]:
589
- """Execute the seed for ``rows`` against a live sqlite, in one transaction.
590
-
591
- Columns come from the live database itself. A refused row rolls the whole
592
- seed back. Returns the row count of every touched table afterwards.
593
- """
594
- statements = sqlite_statements(sqlite_columns(conn), rows, mode)
467
+ def _legacy_counts(database: Path) -> dict[str, int]:
468
+ backing = LegacySQLiteBacking(database)
595
469
  try:
596
- conn.execute("BEGIN")
597
- for statement in statements:
598
- conn.execute(statement)
599
- conn.commit()
600
- except sqlite3.Error as error:
601
- conn.rollback()
602
- fail("sqlite", f"seed refused by the database: {error}", "/rows")
603
- return {table: conn.execute(f"SELECT COUNT(*) FROM {sql_ident(table)}").fetchone()[0] for table in rows}
604
-
605
-
606
- def _sqlite_counts(database: Path) -> dict[str, int]:
607
- conn = sqlite3.connect(database)
608
- try:
609
- return {
610
- table: conn.execute(f"SELECT COUNT(*) FROM {sql_ident(table)}").fetchone()[0]
611
- for table in sqlite_columns(conn)
612
- }
470
+ return backing.counts()
613
471
  finally:
614
- conn.close()
615
-
616
-
617
- # ---------------------------------------------------------------- show
472
+ backing.close()
618
473
 
619
474
 
620
475
  def entity_counts(root: str | Path) -> dict[str, int]:
@@ -628,33 +483,33 @@ def entity_counts(root: str | Path) -> dict[str, int]:
628
483
  if kind not in (KIND_SCHEMA, KIND_MOCK_TOOLS):
629
484
  _refuse_kind(kind, root)
630
485
  if kind == KIND_SCHEMA:
631
- from .snapshot import current_default
486
+ from .snapshot import held_default
632
487
 
633
488
  contract = check_bundle(root)
634
- snapshot = current_default(root, contract)
635
- if snapshot is not None:
636
- return dict(snapshot["entityCounts"])
489
+ held = held_default(root, contract)
490
+ if held is not None:
491
+ return dict(held[1]["entityCounts"])
637
492
  from .._materialize import materialize_template
638
493
 
639
494
  with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
640
495
  database = Path(scratch) / "world.db"
641
496
  materialize_template(root, database, require_sql=kind == KIND_MOCK_TOOLS)
642
497
  if kind == KIND_SCHEMA:
643
- with SQLiteStore.open(database, contract) as store:
498
+ with WorldDatabase.open(database, contract) as store:
644
499
  return store.counts()
645
- return _sqlite_counts(database)
500
+ return _legacy_counts(database)
646
501
 
647
502
 
648
503
  @contextmanager
649
- def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Iterator[SQLiteStore]:
504
+ def schema_store(root: Path, contract: Any, scenario: str | None = None) -> Iterator[WorldDatabase]:
650
505
  """The rows of a schema world, read only: the compiled snapshot opened in place
651
- when it is current (``db/state.sqlite`` is this data through this contract),
652
- else the data files applied in a scratch copy, as a served world would. With
653
- ``scenario``, the rows a session opened in ``scenarios/<scenario>/`` starts
654
- from: the template with the scenario's seed overlay replacing whole entities,
655
- built in scratch (the compiled ``scenarios/<name>/state.sqlite`` is not
506
+ when it is current (the default snapshot is this data through this contract, or a
507
+ pinned tree's copy in the snapshot store), else the data files applied in a scratch
508
+ copy, as a served world would. With ``scenario``, the rows a session opened in
509
+ ``scenarios/<scenario>/`` starts from: the template with the scenario's seed overlay
510
+ replacing whole entities, built in scratch (the compiled scenario snapshot is not
656
511
  consulted, so the answer is the tree as it is now)."""
657
- from .snapshot import SCENARIOS, current_default, default_snapshot, scenario_dirs, scenario_overlay
512
+ from .snapshot import SCENARIOS, held_default, scenario_dirs, scenario_overlay
658
513
 
659
514
  overlay: dict[str, list] | None = None
660
515
  if scenario is not None:
@@ -666,8 +521,9 @@ def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Ite
666
521
  f"/{SCENARIOS}/{scenario}",
667
522
  )
668
523
  overlay = scenario_overlay(scenarios[scenario])
669
- if overlay is None and current_default(root, contract) is not None:
670
- with SQLiteStore.open(default_snapshot(root), contract, audit=False) as store:
524
+ held = held_default(root, contract) if overlay is None else None
525
+ if held is not None:
526
+ with WorldDatabase.open(held[0], contract, audit=False) as store:
671
527
  yield store
672
528
  return
673
529
  from .._materialize import materialize_template
@@ -675,7 +531,7 @@ def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Ite
675
531
  with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
676
532
  database = Path(scratch) / "world.db"
677
533
  materialize_template(root, database)
678
- with SQLiteStore.open(database, contract) as store:
534
+ with WorldDatabase.open(database, contract) as store:
679
535
  if overlay:
680
536
  store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{scenario}"))
681
537
  yield store
@@ -699,15 +555,11 @@ def _table_rows(root: Path, entity: str) -> tuple[list[dict], list[str]]:
699
555
  with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
700
556
  database = Path(scratch) / "world.db"
701
557
  materialize_template(root, database, require_sql=True)
702
- conn = sqlite3.connect(database)
558
+ backing = LegacySQLiteBacking(database)
703
559
  try:
704
- columns = sqlite_columns(conn)
705
- if entity not in columns:
706
- fail("table", f"No table {entity}; tables: {', '.join(sorted(columns)) or 'none'}", f"/{entity}")
707
- conn.row_factory = sqlite3.Row
708
- return [dict(row) for row in conn.execute(f"SELECT * FROM {sql_ident(entity)}")], columns[entity]
560
+ return backing.table_rows(entity)
709
561
  finally:
710
- conn.close()
562
+ backing.close()
711
563
 
712
564
 
713
565
  def _seed_rows(root: Path, entity: str) -> list[dict]:
@@ -729,7 +581,7 @@ def entity_rows(root: str | Path, entity: str) -> list[dict]:
729
581
  return _table_rows(root, entity)[0]
730
582
  if kind != KIND_SCHEMA:
731
583
  _refuse_kind(kind, root)
732
- with _schema_store(root, _schema_contract(root, entity)) as store:
584
+ with schema_store(root, _schema_contract(root, entity)) as store:
733
585
  return list(store.list(entity))
734
586
 
735
587
 
@@ -738,9 +590,7 @@ def _listed(value: Any) -> list:
738
590
 
739
591
 
740
592
  def _filtered(rows: list[dict], fields: list[str], wanted: dict[str, list[str]], entity: str) -> list[dict]:
741
- """The rows every ``wanted`` field of which matches, by the store's rule (``_matches``)."""
742
- from .api import _matches
743
-
593
+ """The rows every ``wanted`` field of which matches, by the store's rule (:class:`RowMatch`)."""
744
594
  unknown = sorted(set(wanted) - set(fields))
745
595
  if unknown:
746
596
  fail(
@@ -749,8 +599,7 @@ def _filtered(rows: list[dict], fields: list[str], wanted: dict[str, list[str]],
749
599
  entity=entity,
750
600
  fields=sorted(fields),
751
601
  )
752
- filters = {field: field for field in wanted}
753
- return [row for row in rows if _matches(row, filters, wanted)]
602
+ return RowMatch.on_fields(wanted).select(rows)
754
603
 
755
604
 
756
605
  def entity_query(
@@ -789,10 +638,8 @@ def entity_query(
789
638
  limit = min(limit, QUERY_LIMIT)
790
639
  resolved: dict[str, list] | None = None
791
640
  wanted = {field: _listed(values) for field, values in where.items()}
792
- # A cross-world mapping normalises the plaintext BEFORE this world's redaction
793
- # (``[dependencies.<alias>.entities]`` ``transform``): ``where_transform`` maps a
794
- # field to its chain, validated up front so an unknown name refuses the whole
795
- # query rather than one field silently matching nothing.
641
+ # A cross-world ``transform`` normalises the plaintext BEFORE this world's redaction; chains are
642
+ # validated up front so an unknown name refuses the query instead of one field matching nothing.
796
643
  from .transforms import apply_transforms, parse_transform
797
644
 
798
645
  chains: dict[str, list[str]] = {}
@@ -827,7 +674,7 @@ def entity_query(
827
674
  wanted = {field: [str(value) for value in values] for field, values in wanted.items()}
828
675
  kind = world_kind(root)
829
676
  if kind == KIND_SCHEMA:
830
- with _schema_store(root, _schema_contract(root, entity), scenario) as store:
677
+ with schema_store(root, _schema_contract(root, entity), scenario) as store:
831
678
  rows, total = store.list_where(entity, wanted, limit, offset)
832
679
  else:
833
680
  if scenario is not None:
@@ -992,8 +839,5 @@ __all__ = [
992
839
  "import_rows",
993
840
  "parse_rows",
994
841
  "seed_batch",
995
- "seed_sqlite",
996
- "sqlite_columns",
997
- "sqlite_statements",
998
842
  "world_kind",
999
843
  ]
@@ -0,0 +1,24 @@
1
+ """The world database: a contract-bound store with engine subclasses."""
2
+
3
+ from .base import WorldDatabase
4
+ from .budget import QueryBudget
5
+ from .connections import IngestConnection, ReadConnection, ReadCursor
6
+ from .guard import Access, AccessKind, Guard
7
+ from .matching import RowMatch
8
+ from .rows import Row, Rows
9
+ from .sqlite import SQLiteWorldDatabase
10
+
11
+ __all__ = [
12
+ "Access",
13
+ "AccessKind",
14
+ "Guard",
15
+ "IngestConnection",
16
+ "QueryBudget",
17
+ "ReadConnection",
18
+ "ReadCursor",
19
+ "Row",
20
+ "Rows",
21
+ "RowMatch",
22
+ "SQLiteWorldDatabase",
23
+ "WorldDatabase",
24
+ ]
@@ -0,0 +1,121 @@
1
+ """Persisted rows checked against the contract: payload, key identity, references."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import TYPE_CHECKING
7
+
8
+ from ..compiler import payload_value, quote
9
+ from ..validation import ContractError, fail, parse_json, validate
10
+
11
+ if TYPE_CHECKING:
12
+ from .base import WorldDatabase
13
+
14
+
15
+ class ContractAudit:
16
+ """Reads a database's rows as the store wrote them and names every row the
17
+ contract refuses, with the batch that last wrote it."""
18
+
19
+ def __init__(self, database: "WorldDatabase") -> None:
20
+ self._database = database
21
+
22
+ def violations(self, limit: int | None = None) -> list[ContractError]:
23
+ """Contract violations in persisted state, at most ``limit`` of them."""
24
+ database = self._database
25
+ found: list[ContractError] = []
26
+ with database._trusted():
27
+ rows = [
28
+ (name, entity, row)
29
+ for name, entity in database.contract.entities.items()
30
+ for row in database._execute(f"SELECT _key, payload FROM {quote(name)} ORDER BY _key").fetchall()
31
+ ]
32
+ for name, entity, row in rows:
33
+ error = self.row(name, entity, row)
34
+ if error is None:
35
+ continue
36
+ found.append(error)
37
+ if limit is not None and len(found) >= limit:
38
+ return found
39
+ with database._trusted():
40
+ if database._relationships_broken():
41
+ found.append(self.relationship_failure())
42
+ return found
43
+
44
+ def touched(self, touched: set[tuple[str, str]]) -> None:
45
+ """The commit-time audit of one transaction: the rows it wrote, re-read from
46
+ storage and checked like a full audit, plus the engine's own relationship check
47
+ over the whole state (a delete that orphans an untouched row shows up there).
48
+ Rows the transaction never touched cannot have changed, so a transaction's
49
+ cost follows its size, not the size of the world."""
50
+ database = self._database
51
+ with database._trusted():
52
+ for name, key in sorted(touched):
53
+ self._touched_row(name, key)
54
+ if database._relationships_broken():
55
+ raise self.relationship_failure()
56
+
57
+ def _touched_row(self, name: str, key: str) -> None:
58
+ database = self._database
59
+ row = database._execute(f"SELECT _key, payload FROM {quote(name)} WHERE _key = ?", (key,)).fetchone()
60
+ if row is None:
61
+ return
62
+ error = self.row(name, database.contract.entities[name], row)
63
+ if error is not None:
64
+ raise error
65
+
66
+ def relationship_failure(self) -> ContractError:
67
+ return ContractError("foreign_key", f"Persisted {self._database.engine_label} relationship check failed")
68
+
69
+ def row(self, name: str, entity: dict, row) -> ContractError | None:
70
+ """The contract violation for one persisted row, with lineage context."""
71
+ try:
72
+ value = parse_json(row[1], f"{entity['location']}/schema")
73
+ except ContractError as error:
74
+ return error.contextualize(entity=name)
75
+ except (ValueError, TypeError):
76
+ return ContractError("state_payload", "Stored entity payload is not JSON", entity=name)
77
+ try:
78
+ validate(entity["schema"], value, location=f"{entity['location']}/schema")
79
+ if self._database._key(name, value) != row[0]:
80
+ fail("key", "Stored key differs from payload identity", entity=name)
81
+ for rel in entity["relationships"]:
82
+ self._relationship(entity, rel, value)
83
+ except ContractError as error:
84
+ return error.contextualize(**self._context(name, row[0]))
85
+ return None
86
+
87
+ def _relationship(self, entity: dict, rel: dict, value: dict) -> None:
88
+ keys = [payload_value(value, field) for field in rel["fields"]]
89
+ if all(k is None for k in keys):
90
+ return
91
+ if any(k is None for k in keys):
92
+ fail(
93
+ "foreign_key",
94
+ "Composite references must be wholly present or wholly absent/null",
95
+ f"{entity['location']}/relationships",
96
+ )
97
+ target = self._database.contract.entities[rel["target"]]
98
+ target_key = dict(zip(target["primary_key"], keys))
99
+ if self._database.get(rel["target"], target_key) is None:
100
+ fail(
101
+ "foreign_key",
102
+ "Relationship target does not exist; incomplete evidence requires an explicit mapping decision",
103
+ f"{entity['location']}/relationships",
104
+ )
105
+
106
+ def _context(self, name: str, stored_key: str) -> dict:
107
+ database = self._database
108
+ try:
109
+ diagnostic_key = json.loads(stored_key)
110
+ except (ValueError, TypeError):
111
+ diagnostic_key = None
112
+ context = {"entity": name, "key": diagnostic_key}
113
+ with database._trusted():
114
+ lineage = database._execute(
115
+ "SELECT batch, ordinal, source FROM _gateway_lineage "
116
+ f"WHERE entity=? AND key=? ORDER BY {database.insertion_order} DESC LIMIT 1",
117
+ (name, stored_key),
118
+ ).fetchone()
119
+ if lineage:
120
+ context.update(batch=lineage[0], index=lineage[1], source=json.loads(lineage[2]))
121
+ return context