@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -18,8 +18,9 @@ way that kind seeds:
18
18
  are generated or per-task; the import refuses and names what to edit.
19
19
 
20
20
  The same row planning seeds a *live* session: :func:`seed_batch` builds the
21
- batch a running store applies atomically, and :func:`seed_sqlite` executes
22
- the inserts against a task's live sqlite.
21
+ batch a running store applies atomically, and
22
+ :func:`~gatewaysdk.environment.legacy_sqlite.seed_sqlite` executes the inserts
23
+ against a mock-tools task's live database.
23
24
  """
24
25
 
25
26
  from __future__ import annotations
@@ -28,22 +29,22 @@ import hashlib
28
29
  import json
29
30
  import re
30
31
  import shutil
31
- import sqlite3
32
32
  import tempfile
33
33
  import uuid
34
34
  from contextlib import contextmanager
35
35
  from pathlib import Path
36
36
  from typing import Any, Iterable, Iterator
37
37
 
38
+ from ..legacy_sqlite import MODES, LegacySQLiteBacking, schema_columns, seed_statements
38
39
  from ..world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, SCHEMA_MANIFEST_KIND, World
39
40
  from .cast import EntityCasters
40
41
  from .compiler import bundle_mode, check_bundle, compile_bundle
42
+ from .primary_key import PrimaryKey
41
43
  from .proof import SCRATCH_IGNORE, RowProof
42
- from .store import SQLiteStore
44
+ from .database import RowMatch, WorldDatabase
43
45
  from .tree_import import refuse_unpublishable_journal
44
- from .validation import ContractError, digest, encoded, fail, journal_paths, read_json, validate
46
+ from .validation import ContractError, digest, encoded, fail, journal_paths, read_json
45
47
 
46
- MODES = ("append", "replace")
47
48
  MERGES = ("upsert", "fill")
48
49
  #: The inline lane's caps (the platform's seed and import calls share them): a local
49
50
  #: directory import above either is refused; larger data is a platform batch.
@@ -58,9 +59,6 @@ MAPPING = {"id": "data-import", "version": "1"}
58
59
  Rows = dict[str, list[dict]]
59
60
 
60
61
 
61
- # ---------------------------------------------------------------- rows files
62
-
63
-
64
62
  def parse_rows(
65
63
  paths: Iterable[str | Path],
66
64
  *,
@@ -94,9 +92,6 @@ def parse_rows(
94
92
  return rows
95
93
 
96
94
 
97
- # ---------------------------------------------------------------- world kinds
98
-
99
-
100
95
  def world_kind(root: str | Path) -> str:
101
96
  """The kind the worlds paths report (``World.from_dir(...).reported_kind``):
102
97
  ``schema``, ``mock-tools``, ``api``, ``harbor``, ``verifiers`` or
@@ -131,9 +126,6 @@ def _refuse_kind(kind: str, root: Path) -> None:
131
126
  )
132
127
 
133
128
 
134
- # ---------------------------------------------------------------- import
135
-
136
-
137
129
  def import_rows(
138
130
  root: str | Path,
139
131
  rows: Rows,
@@ -201,7 +193,7 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
201
193
  the row's index in the file, carrying the full ``refusals`` list — each
202
194
  naming its entity, index, primary key and field — the shape the batch lane
203
195
  reports."""
204
- from .batch import shape_refusal
196
+ from .batch import raise_shape_refusal, shape_refusal
205
197
 
206
198
  contract = compile_bundle(root, snapshots=False)
207
199
  refusals = [
@@ -214,22 +206,10 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
214
206
  return
215
207
  first = refusals[0]
216
208
  entity, index = first["entity"], first["index"]
217
- row = rows[entity][index]
218
209
  total = sum(len(listed) for listed in rows.values())
219
210
  summary = f"{len(refusals)} of {total} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
220
211
  context = {"entity": entity, "index": index, "refused": len(refusals), "rows": total, "refusals": refusals}
221
- if entity not in contract.entities:
222
- fail("entity", summary + first["detail"], f"/{entity}", **context)
223
- if not isinstance(row, dict):
224
- fail("rows", summary + first["detail"], first["pointer"], **context)
225
- spec = contract.entities[entity]
226
- if any(name not in row for name in spec["primary_key"]):
227
- fail("primary_key", summary + first["detail"], first["pointer"], **context)
228
- try:
229
- validate(spec["schema"], row, location=f"{spec['location']}/schema")
230
- except ContractError as error:
231
- raise error.contextualize(message=summary + error.detail["message"], **context)
232
- fail("rows", summary + first["detail"], first["pointer"], **context)
212
+ raise_shape_refusal(contract, first, rows[entity][index], summary, context, f"/{entity}")
233
213
 
234
214
 
235
215
  def _merged(root: Path, rows: Rows, merge: str) -> Rows:
@@ -266,16 +246,9 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
266
246
  the next journal instead: a replace deletes rows those journals hold, and
267
247
  an append upserts against them, rather than being overwritten on replay.
268
248
  """
269
- from .snapshot import pin_path
249
+ from .snapshot import refuse_pinned_rows
270
250
 
271
- if pin_path(root).is_file():
272
- fail(
273
- "snapshot_pinned",
274
- "This world's data is a hosted snapshot (db/snapshot.json), which its data files "
275
- "cannot reproduce: import rows through the platform (`gateway worlds data import "
276
- "<slug>` or import_world_data) instead of into the checkout",
277
- "/db/snapshot.json",
278
- )
251
+ refuse_pinned_rows(root)
279
252
  contract = compile_bundle(root)
280
253
  journals = journal_paths(root)
281
254
  initial_path = root / "data/initial.json"
@@ -287,8 +260,7 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
287
260
  if not fresh:
288
261
  return {"kind": KIND_SCHEMA, "batch": prior_id, "entity_counts": entity_counts(root), "changed": False}
289
262
  if mode == "replace" and not refreshed:
290
- # Folded into initial.json, a replace is written compactly: the named
291
- # entities' old mutations are dropped and the rows inserted afresh.
263
+ # Folded into initial.json: the named entities' old mutations are dropped and the rows inserted afresh.
292
264
  prior = [m for m in prior if m.get("entity") not in rows]
293
265
  fresh = _plan(contract, rows, mode, _held(contract, prior))
294
266
  match = re.search(r"\.v(\d+)$", prior_id)
@@ -323,15 +295,15 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
323
295
  (staged / written).parent.mkdir(parents=True, exist_ok=True)
324
296
  (staged / written).write_bytes(payload)
325
297
  # Building the staged snapshot is the proof: every row goes through the store.
326
- counts = build_default(staged, compile_bundle(staged, snapshots=False))["entityCounts"]
298
+ contract = compile_bundle(staged, snapshots=False)
299
+ counts = build_default(staged, contract)["entityCounts"]
327
300
  (root / written).parent.mkdir(parents=True, exist_ok=True)
328
301
  (root / written).write_bytes(payload)
329
- # The proven snapshot moves into the world; compile finds it current and only
330
- # rebuilds the scenario snapshots on top of it.
331
- target = default_snapshot(root)
302
+ # The proven snapshot moves in; compile then finds it current and only rebuilds the scenarios.
303
+ target = default_snapshot(root, contract)
332
304
  target.parent.mkdir(parents=True, exist_ok=True)
333
- shutil.move(str(default_snapshot(staged)), str(target))
334
- shutil.move(str(sidecar_path(default_snapshot(staged))), str(sidecar_path(target)))
305
+ shutil.move(str(default_snapshot(staged, contract)), str(target))
306
+ shutil.move(str(sidecar_path(default_snapshot(staged, contract))), str(sidecar_path(target)))
335
307
  compile_bundle(root)
336
308
  check_bundle(root)
337
309
  return {"kind": KIND_SCHEMA, "batch": batch["id"], "entity_counts": counts, "changed": True}
@@ -363,10 +335,7 @@ def _held(contract: Any, mutations: list[dict]) -> dict[tuple[str, tuple], dict]
363
335
 
364
336
 
365
337
  def _key(contract: Any, entity: str, row: dict) -> tuple | None:
366
- fields = contract.entities.get(entity, {}).get("primary_key", [])
367
- if not fields or any(field not in row for field in fields):
368
- return None
369
- return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
338
+ return PrimaryKey.in_contract(contract, entity).identity(row)
370
339
 
371
340
 
372
341
  def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], dict]) -> list[dict]:
@@ -389,12 +358,11 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
389
358
  }
390
359
  for (entity, key), value in list(held.items()):
391
360
  if entity in rows and (entity, key) not in incoming:
392
- fields = contract.entities[entity]["primary_key"]
393
361
  mutations.append(
394
362
  {
395
363
  "op": "delete",
396
364
  "entity": entity,
397
- "key": {field: value[field] for field in fields},
365
+ "key": PrimaryKey.in_contract(contract, entity).key(value),
398
366
  "source": {"record": f"data-import:{entity}:replace"},
399
367
  }
400
368
  )
@@ -404,13 +372,13 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
404
372
  fail(
405
373
  "entity", f"Unknown entity {entity}; the world has {', '.join(sorted(contract.entities))}", f"/{entity}"
406
374
  )
407
- fields = contract.entities[entity]["primary_key"]
375
+ primary = PrimaryKey.in_contract(contract, entity)
408
376
  for index, row in enumerate(listed):
409
377
  if not isinstance(row, dict):
410
378
  fail("rows", "A row must be an object", f"/{entity}/{index}")
411
- missing = [field for field in fields if field not in row]
412
- if missing:
413
- fail("primary_key", f"Row is missing primary key field {missing[0]!r}", f"/{entity}/{index}")
379
+ missing = primary.missing(row)
380
+ if missing is not None:
381
+ fail("primary_key", f"Row is missing primary key field {missing!r}", f"/{entity}/{index}")
414
382
  key = _key(contract, entity, row)
415
383
  assert key is not None
416
384
  existing = held.get((entity, key))
@@ -424,7 +392,7 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
424
392
  "source": {"record": f"data-import:{entity}:{identity}"},
425
393
  }
426
394
  if existing is not None:
427
- mutation["key"] = {field: row[field] for field in fields}
395
+ mutation["key"] = primary.key(row)
428
396
  mutations.append(mutation)
429
397
  held[(entity, key)] = row
430
398
  return mutations
@@ -445,22 +413,18 @@ def seed_batch(store: Any, contract: Any, rows: Rows, mode: str = "append", *, b
445
413
  if entity not in contract.entities:
446
414
  continue
447
415
  if mode == "replace":
448
- # Every held row of a replaced entity goes; the plan needs them all.
449
416
  for value in store.list(entity):
450
417
  key = _key(contract, entity, value)
451
418
  if key is not None:
452
419
  held[(entity, key)] = value
453
420
  continue
454
- # An append only needs the rows at the keys it names, however large the entity.
455
- fields = contract.entities[entity]["primary_key"]
421
+ primary = PrimaryKey.in_contract(contract, entity)
456
422
  for row in listed:
457
- if not isinstance(row, dict) or any(field not in row for field in fields):
458
- continue
459
- key = _key(contract, entity, row)
423
+ key = _key(contract, entity, row) if isinstance(row, dict) else None
460
424
  if key is None or (entity, key) in held:
461
425
  continue
462
426
  try:
463
- value = store.get(entity, {field: row[field] for field in fields})
427
+ value = store.get(entity, primary.key(row))
464
428
  except ContractError:
465
429
  continue # a key the contract refuses is refused by the plan, with the field named
466
430
  if value is not None:
@@ -475,20 +439,11 @@ def seed_batch(store: Any, contract: Any, rows: Rows, mode: str = "append", *, b
475
439
  }
476
440
 
477
441
 
478
- # ---------------------------------------------------------------- mock-tools
479
-
480
-
481
442
  def _import_mock_tools(root: Path, rows: Rows, mode: str) -> dict:
482
443
  schema = root / "db/schema.sql"
483
444
  if not schema.is_file():
484
445
  fail("schema", "mock-tools world has no db/schema.sql to read tables from", "db/schema.sql")
485
- with sqlite3.connect(":memory:") as probe:
486
- try:
487
- probe.executescript(schema.read_text(encoding="utf-8"))
488
- except sqlite3.Error as error:
489
- fail("schema", f"db/schema.sql does not load: {error}", "db/schema.sql")
490
- columns = sqlite_columns(probe)
491
- statements = sqlite_statements(columns, rows, mode)
446
+ statements = seed_statements(schema_columns(schema), rows, mode)
492
447
  if not statements:
493
448
  return {"kind": KIND_MOCK_TOOLS, "entity_counts": entity_counts(root), "changed": False}
494
449
  seed = root / "db/seed.sql"
@@ -504,117 +459,17 @@ def _import_mock_tools(root: Path, rows: Rows, mode: str) -> dict:
504
459
  (staged / "db/seed.sql").write_text(appended, encoding="utf-8")
505
460
  database = Path(scratch) / "world.db"
506
461
  materialize_template(staged, database, require_sql=True)
507
- counts = _sqlite_counts(database)
462
+ counts = _legacy_counts(database)
508
463
  seed.write_text(appended, encoding="utf-8")
509
464
  return {"kind": KIND_MOCK_TOOLS, "entity_counts": counts, "changed": True, "statements": len(statements)}
510
465
 
511
466
 
512
- def sqlite_columns(conn: sqlite3.Connection) -> dict[str, list[str]]:
513
- """Every user table's columns, in declaration order."""
514
- tables = [
515
- row[0]
516
- for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")
517
- ]
518
- return {table: [row[1] for row in conn.execute(f"PRAGMA table_info({sql_ident(table)})")] for table in tables}
519
-
520
-
521
- def sql_ident(name: str) -> str:
522
- """A SQLite identifier, quoted: a table or column named by a schema is a
523
- name, never SQL, whatever characters it carries."""
524
- return '"' + name.replace('"', '""') + '"'
525
-
526
-
527
- def _check_sqlite_rows(columns: dict[str, list[str]], rows: Rows) -> None:
528
- for table, listed in rows.items():
529
- if table not in columns:
530
- fail(
531
- "table",
532
- f"No table {table} in db/schema.sql; tables: {', '.join(sorted(columns)) or 'none'}",
533
- f"/{table}",
534
- )
535
- for index, row in enumerate(listed):
536
- if not isinstance(row, dict):
537
- fail("rows", "A row must be an object", f"/{table}/{index}")
538
- for column, value in row.items():
539
- if column not in columns[table]:
540
- fail(
541
- "column",
542
- f"Column {column} is not in table {table}; columns: {', '.join(columns[table])}",
543
- f"/{table}/{index}/{column}",
544
- )
545
- if not isinstance(value, (str, int, float, bool, type(None))):
546
- fail(
547
- "value",
548
- f"Column {column} of {table} takes a scalar; encode nested values as text yourself",
549
- f"/{table}/{index}/{column}",
550
- )
551
-
552
-
553
- def sqlite_statements(columns: dict[str, list[str]], rows: Rows, mode: str = "append") -> list[str]:
554
- """SQL text that seeds ``rows`` into tables with these ``columns``.
555
-
556
- ``append`` writes ``INSERT OR REPLACE`` (an upsert by the table's own
557
- primary key); ``replace`` prefixes ``DELETE FROM`` for each named table and
558
- writes plain inserts.
559
- """
560
- if mode not in MODES:
561
- fail("mode", f"mode must be one of {', '.join(MODES)}", "/mode")
562
- _check_sqlite_rows(columns, rows)
563
- statements: list[str] = []
564
- verb = "INSERT" if mode == "replace" else "INSERT OR REPLACE"
565
- for table, listed in rows.items():
566
- if mode == "replace":
567
- statements.append(f"DELETE FROM {sql_ident(table)};")
568
- for row in listed:
569
- if not row:
570
- statements.append(f"{verb} INTO {sql_ident(table)} DEFAULT VALUES;")
571
- continue
572
- names = ", ".join(sql_ident(column) for column in row)
573
- values = ", ".join(_sql_literal(value) for value in row.values())
574
- statements.append(f"{verb} INTO {sql_ident(table)} ({names}) VALUES ({values});")
575
- return statements
576
-
577
-
578
- def _sql_literal(value: Any) -> str:
579
- if value is None:
580
- return "NULL"
581
- if isinstance(value, bool):
582
- return "1" if value else "0"
583
- if isinstance(value, (int, float)):
584
- return repr(value)
585
- return "'" + str(value).replace("'", "''") + "'"
586
-
587
-
588
- def seed_sqlite(conn: sqlite3.Connection, rows: Rows, mode: str = "append") -> dict[str, int]:
589
- """Execute the seed for ``rows`` against a live sqlite, in one transaction.
590
-
591
- Columns come from the live database itself. A refused row rolls the whole
592
- seed back. Returns the row count of every touched table afterwards.
593
- """
594
- statements = sqlite_statements(sqlite_columns(conn), rows, mode)
467
+ def _legacy_counts(database: Path) -> dict[str, int]:
468
+ backing = LegacySQLiteBacking(database)
595
469
  try:
596
- conn.execute("BEGIN")
597
- for statement in statements:
598
- conn.execute(statement)
599
- conn.commit()
600
- except sqlite3.Error as error:
601
- conn.rollback()
602
- fail("sqlite", f"seed refused by the database: {error}", "/rows")
603
- return {table: conn.execute(f"SELECT COUNT(*) FROM {sql_ident(table)}").fetchone()[0] for table in rows}
604
-
605
-
606
- def _sqlite_counts(database: Path) -> dict[str, int]:
607
- conn = sqlite3.connect(database)
608
- try:
609
- return {
610
- table: conn.execute(f"SELECT COUNT(*) FROM {sql_ident(table)}").fetchone()[0]
611
- for table in sqlite_columns(conn)
612
- }
470
+ return backing.counts()
613
471
  finally:
614
- conn.close()
615
-
616
-
617
- # ---------------------------------------------------------------- show
472
+ backing.close()
618
473
 
619
474
 
620
475
  def entity_counts(root: str | Path) -> dict[str, int]:
@@ -640,19 +495,19 @@ def entity_counts(root: str | Path) -> dict[str, int]:
640
495
  database = Path(scratch) / "world.db"
641
496
  materialize_template(root, database, require_sql=kind == KIND_MOCK_TOOLS)
642
497
  if kind == KIND_SCHEMA:
643
- with SQLiteStore.open(database, contract) as store:
498
+ with WorldDatabase.open(database, contract) as store:
644
499
  return store.counts()
645
- return _sqlite_counts(database)
500
+ return _legacy_counts(database)
646
501
 
647
502
 
648
503
  @contextmanager
649
- def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Iterator[SQLiteStore]:
504
+ def schema_store(root: Path, contract: Any, scenario: str | None = None) -> Iterator[WorldDatabase]:
650
505
  """The rows of a schema world, read only: the compiled snapshot opened in place
651
- when it is current (``db/state.sqlite`` is this data through this contract),
506
+ when it is current (the default snapshot is this data through this contract),
652
507
  else the data files applied in a scratch copy, as a served world would. With
653
508
  ``scenario``, the rows a session opened in ``scenarios/<scenario>/`` starts
654
509
  from: the template with the scenario's seed overlay replacing whole entities,
655
- built in scratch (the compiled ``scenarios/<name>/state.sqlite`` is not
510
+ built in scratch (the compiled scenario snapshot is not
656
511
  consulted, so the answer is the tree as it is now)."""
657
512
  from .snapshot import SCENARIOS, current_default, default_snapshot, scenario_dirs, scenario_overlay
658
513
 
@@ -667,7 +522,7 @@ def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Ite
667
522
  )
668
523
  overlay = scenario_overlay(scenarios[scenario])
669
524
  if overlay is None and current_default(root, contract) is not None:
670
- with SQLiteStore.open(default_snapshot(root), contract, audit=False) as store:
525
+ with WorldDatabase.open(default_snapshot(root, contract), contract, audit=False) as store:
671
526
  yield store
672
527
  return
673
528
  from .._materialize import materialize_template
@@ -675,7 +530,7 @@ def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Ite
675
530
  with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
676
531
  database = Path(scratch) / "world.db"
677
532
  materialize_template(root, database)
678
- with SQLiteStore.open(database, contract) as store:
533
+ with WorldDatabase.open(database, contract) as store:
679
534
  if overlay:
680
535
  store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{scenario}"))
681
536
  yield store
@@ -699,15 +554,11 @@ def _table_rows(root: Path, entity: str) -> tuple[list[dict], list[str]]:
699
554
  with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
700
555
  database = Path(scratch) / "world.db"
701
556
  materialize_template(root, database, require_sql=True)
702
- conn = sqlite3.connect(database)
557
+ backing = LegacySQLiteBacking(database)
703
558
  try:
704
- columns = sqlite_columns(conn)
705
- if entity not in columns:
706
- fail("table", f"No table {entity}; tables: {', '.join(sorted(columns)) or 'none'}", f"/{entity}")
707
- conn.row_factory = sqlite3.Row
708
- return [dict(row) for row in conn.execute(f"SELECT * FROM {sql_ident(entity)}")], columns[entity]
559
+ return backing.table_rows(entity)
709
560
  finally:
710
- conn.close()
561
+ backing.close()
711
562
 
712
563
 
713
564
  def _seed_rows(root: Path, entity: str) -> list[dict]:
@@ -729,7 +580,7 @@ def entity_rows(root: str | Path, entity: str) -> list[dict]:
729
580
  return _table_rows(root, entity)[0]
730
581
  if kind != KIND_SCHEMA:
731
582
  _refuse_kind(kind, root)
732
- with _schema_store(root, _schema_contract(root, entity)) as store:
583
+ with schema_store(root, _schema_contract(root, entity)) as store:
733
584
  return list(store.list(entity))
734
585
 
735
586
 
@@ -738,9 +589,7 @@ def _listed(value: Any) -> list:
738
589
 
739
590
 
740
591
  def _filtered(rows: list[dict], fields: list[str], wanted: dict[str, list[str]], entity: str) -> list[dict]:
741
- """The rows every ``wanted`` field of which matches, by the store's rule (``_matches``)."""
742
- from .api import _matches
743
-
592
+ """The rows every ``wanted`` field of which matches, by the store's rule (:class:`RowMatch`)."""
744
593
  unknown = sorted(set(wanted) - set(fields))
745
594
  if unknown:
746
595
  fail(
@@ -749,8 +598,7 @@ def _filtered(rows: list[dict], fields: list[str], wanted: dict[str, list[str]],
749
598
  entity=entity,
750
599
  fields=sorted(fields),
751
600
  )
752
- filters = {field: field for field in wanted}
753
- return [row for row in rows if _matches(row, filters, wanted)]
601
+ return RowMatch.on_fields(wanted).select(rows)
754
602
 
755
603
 
756
604
  def entity_query(
@@ -789,10 +637,8 @@ def entity_query(
789
637
  limit = min(limit, QUERY_LIMIT)
790
638
  resolved: dict[str, list] | None = None
791
639
  wanted = {field: _listed(values) for field, values in where.items()}
792
- # A cross-world mapping normalises the plaintext BEFORE this world's redaction
793
- # (``[dependencies.<alias>.entities]`` ``transform``): ``where_transform`` maps a
794
- # field to its chain, validated up front so an unknown name refuses the whole
795
- # query rather than one field silently matching nothing.
640
+ # A cross-world ``transform`` normalises the plaintext BEFORE this world's redaction; chains are
641
+ # validated up front so an unknown name refuses the query instead of one field matching nothing.
796
642
  from .transforms import apply_transforms, parse_transform
797
643
 
798
644
  chains: dict[str, list[str]] = {}
@@ -827,7 +673,7 @@ def entity_query(
827
673
  wanted = {field: [str(value) for value in values] for field, values in wanted.items()}
828
674
  kind = world_kind(root)
829
675
  if kind == KIND_SCHEMA:
830
- with _schema_store(root, _schema_contract(root, entity), scenario) as store:
676
+ with schema_store(root, _schema_contract(root, entity), scenario) as store:
831
677
  rows, total = store.list_where(entity, wanted, limit, offset)
832
678
  else:
833
679
  if scenario is not None:
@@ -992,8 +838,5 @@ __all__ = [
992
838
  "import_rows",
993
839
  "parse_rows",
994
840
  "seed_batch",
995
- "seed_sqlite",
996
- "sqlite_columns",
997
- "sqlite_statements",
998
841
  "world_kind",
999
842
  ]
@@ -0,0 +1,24 @@
1
+ """The world database: a contract-bound store with engine subclasses."""
2
+
3
+ from .base import WorldDatabase
4
+ from .budget import QueryBudget
5
+ from .connections import IngestConnection, ReadConnection, ReadCursor
6
+ from .guard import Access, AccessKind, Guard
7
+ from .matching import RowMatch
8
+ from .rows import Row, Rows
9
+ from .sqlite import SQLiteWorldDatabase
10
+
11
+ __all__ = [
12
+ "Access",
13
+ "AccessKind",
14
+ "Guard",
15
+ "IngestConnection",
16
+ "QueryBudget",
17
+ "ReadConnection",
18
+ "ReadCursor",
19
+ "Row",
20
+ "Rows",
21
+ "RowMatch",
22
+ "SQLiteWorldDatabase",
23
+ "WorldDatabase",
24
+ ]
@@ -0,0 +1,121 @@
1
+ """Persisted rows checked against the contract: payload, key identity, references."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import TYPE_CHECKING
7
+
8
+ from ..compiler import payload_value, quote
9
+ from ..validation import ContractError, fail, parse_json, validate
10
+
11
+ if TYPE_CHECKING:
12
+ from .base import WorldDatabase
13
+
14
+
15
+ class ContractAudit:
16
+ """Reads a database's rows as the store wrote them and names every row the
17
+ contract refuses, with the batch that last wrote it."""
18
+
19
+ def __init__(self, database: "WorldDatabase") -> None:
20
+ self._database = database
21
+
22
+ def violations(self, limit: int | None = None) -> list[ContractError]:
23
+ """Contract violations in persisted state, at most ``limit`` of them."""
24
+ database = self._database
25
+ found: list[ContractError] = []
26
+ with database._trusted():
27
+ rows = [
28
+ (name, entity, row)
29
+ for name, entity in database.contract.entities.items()
30
+ for row in database._execute(f"SELECT _key, payload FROM {quote(name)} ORDER BY _key").fetchall()
31
+ ]
32
+ for name, entity, row in rows:
33
+ error = self.row(name, entity, row)
34
+ if error is None:
35
+ continue
36
+ found.append(error)
37
+ if limit is not None and len(found) >= limit:
38
+ return found
39
+ with database._trusted():
40
+ if database._relationships_broken():
41
+ found.append(self.relationship_failure())
42
+ return found
43
+
44
+ def touched(self, touched: set[tuple[str, str]]) -> None:
45
+ """The commit-time audit of one transaction: the rows it wrote, re-read from
46
+ storage and checked like a full audit, plus the engine's own relationship check
47
+ over the whole state (a delete that orphans an untouched row shows up there).
48
+ Rows the transaction never touched cannot have changed, so a transaction's
49
+ cost follows its size, not the size of the world."""
50
+ database = self._database
51
+ with database._trusted():
52
+ for name, key in sorted(touched):
53
+ self._touched_row(name, key)
54
+ if database._relationships_broken():
55
+ raise self.relationship_failure()
56
+
57
+ def _touched_row(self, name: str, key: str) -> None:
58
+ database = self._database
59
+ row = database._execute(f"SELECT _key, payload FROM {quote(name)} WHERE _key = ?", (key,)).fetchone()
60
+ if row is None:
61
+ return
62
+ error = self.row(name, database.contract.entities[name], row)
63
+ if error is not None:
64
+ raise error
65
+
66
+ def relationship_failure(self) -> ContractError:
67
+ return ContractError("foreign_key", f"Persisted {self._database.engine_label} relationship check failed")
68
+
69
+ def row(self, name: str, entity: dict, row) -> ContractError | None:
70
+ """The contract violation for one persisted row, with lineage context."""
71
+ try:
72
+ value = parse_json(row[1], f"{entity['location']}/schema")
73
+ except ContractError as error:
74
+ return error.contextualize(entity=name)
75
+ except (ValueError, TypeError):
76
+ return ContractError("state_payload", "Stored entity payload is not JSON", entity=name)
77
+ try:
78
+ validate(entity["schema"], value, location=f"{entity['location']}/schema")
79
+ if self._database._key(name, value) != row[0]:
80
+ fail("key", "Stored key differs from payload identity", entity=name)
81
+ for rel in entity["relationships"]:
82
+ self._relationship(entity, rel, value)
83
+ except ContractError as error:
84
+ return error.contextualize(**self._context(name, row[0]))
85
+ return None
86
+
87
+ def _relationship(self, entity: dict, rel: dict, value: dict) -> None:
88
+ keys = [payload_value(value, field) for field in rel["fields"]]
89
+ if all(k is None for k in keys):
90
+ return
91
+ if any(k is None for k in keys):
92
+ fail(
93
+ "foreign_key",
94
+ "Composite references must be wholly present or wholly absent/null",
95
+ f"{entity['location']}/relationships",
96
+ )
97
+ target = self._database.contract.entities[rel["target"]]
98
+ target_key = dict(zip(target["primary_key"], keys))
99
+ if self._database.get(rel["target"], target_key) is None:
100
+ fail(
101
+ "foreign_key",
102
+ "Relationship target does not exist; incomplete evidence requires an explicit mapping decision",
103
+ f"{entity['location']}/relationships",
104
+ )
105
+
106
+ def _context(self, name: str, stored_key: str) -> dict:
107
+ database = self._database
108
+ try:
109
+ diagnostic_key = json.loads(stored_key)
110
+ except (ValueError, TypeError):
111
+ diagnostic_key = None
112
+ context = {"entity": name, "key": diagnostic_key}
113
+ with database._trusted():
114
+ lineage = database._execute(
115
+ "SELECT batch, ordinal, source FROM _gateway_lineage "
116
+ f"WHERE entity=? AND key=? ORDER BY {database.insertion_order} DESC LIMIT 1",
117
+ (name, stored_key),
118
+ ).fetchone()
119
+ if lineage:
120
+ context.update(batch=lineage[0], index=lineage[1], source=json.loads(lineage[2]))
121
+ return context