@withgateway/sdk 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/cli.js +4304 -2381
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +4266 -2343
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/files/index.js +56 -31
  6. package/dist/files/index.js.map +1 -1
  7. package/dist/files/index.mjs +52 -27
  8. package/dist/files/index.mjs.map +1 -1
  9. package/dist/hooks/index.d.mts +1 -1
  10. package/dist/hooks/index.d.ts +1 -1
  11. package/dist/hooks/index.js +280 -8
  12. package/dist/hooks/index.js.map +1 -1
  13. package/dist/hooks/index.mjs +272 -8
  14. package/dist/hooks/index.mjs.map +1 -1
  15. package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
  16. package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
  17. package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
  18. package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
  19. package/dist/index.d.mts +4 -4
  20. package/dist/index.d.ts +4 -4
  21. package/dist/index.js +1100 -438
  22. package/dist/index.js.map +1 -1
  23. package/dist/index.mjs +1079 -424
  24. package/dist/index.mjs.map +1 -1
  25. package/dist/traces/index.js +45 -20
  26. package/dist/traces/index.js.map +1 -1
  27. package/dist/traces/index.mjs +41 -16
  28. package/dist/traces/index.mjs.map +1 -1
  29. package/dist/tracing/index.d.mts +1 -1
  30. package/dist/tracing/index.d.ts +1 -1
  31. package/dist/tracing/index.js +98 -61
  32. package/dist/tracing/index.js.map +1 -1
  33. package/dist/tracing/index.mjs +98 -61
  34. package/dist/tracing/index.mjs.map +1 -1
  35. package/dist/worlds/index.d.mts +179 -25
  36. package/dist/worlds/index.d.ts +179 -25
  37. package/dist/worlds/index.js +639 -284
  38. package/dist/worlds/index.js.map +1 -1
  39. package/dist/worlds/index.mjs +631 -276
  40. package/dist/worlds/index.mjs.map +1 -1
  41. package/package.json +2 -1
  42. package/runtime/gatewaysdk/environment/_bundle.py +15 -25
  43. package/runtime/gatewaysdk/environment/_materialize.py +53 -9
  44. package/runtime/gatewaysdk/environment/_toml.py +13 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +40 -25
  46. package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
  47. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
  48. package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
  49. package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
  51. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
  52. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
  53. package/runtime/gatewaysdk/environment/schema/data.py +21 -19
  54. package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
  55. package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
  56. package/runtime/gatewaysdk/environment/schema/host.py +156 -546
  57. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
  58. package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
  59. package/runtime/gatewaysdk/environment/schema/local.py +32 -126
  60. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  61. package/runtime/gatewaysdk/environment/schema/platform.py +30 -26
  62. package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
  63. package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
  64. package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
  65. package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
  66. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
  68. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +67 -6
  69. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
  70. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
  71. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
  72. package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
  73. package/runtime/gatewaysdk/environment/schema/tools_world.py +71 -23
  74. package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
  75. package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
  76. package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
  77. package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
  78. package/runtime/gatewaysdk/environment/world.py +507 -0
  79. package/skills/benchmarks-getting-started/SKILL.md +67 -9
  80. package/skills/connector-schema-authoring/connector-worlds.md +8 -1
  81. package/skills/connector-schema-authoring/world-schema.md +1 -1
  82. package/skills/hooks-getting-started/SKILL.md +67 -6
  83. package/skills/world-data-ingestion/SKILL.md +4 -1
  84. package/skills/worlds-getting-started/SKILL.md +11 -11
  85. package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
@@ -35,11 +35,7 @@ from dataclasses import dataclass
35
35
  from http.cookies import SimpleCookie
36
36
  from pathlib import Path
37
37
 
38
- try:
39
- import tomllib
40
- except ModuleNotFoundError: # pragma: no cover - python < 3.11
41
- import tomli as tomllib # type: ignore[no-redef]
42
-
38
+ from .._toml import tomllib
43
39
  from .validation import fail
44
40
 
45
41
  API_BUNDLE = "gateway-api-world.toml"
@@ -34,8 +34,6 @@ import csv
34
34
  import hashlib
35
35
  import io
36
36
  import json
37
- import shutil
38
- import tempfile
39
37
  from pathlib import Path
40
38
  from typing import Any, Callable, Iterable
41
39
 
@@ -377,7 +375,7 @@ def ingest_records(
377
375
  ``batch`` from the import; ``dry_run`` and ``redact`` say how it ran. With
378
376
  any refusal nothing is imported.
379
377
  """
380
- from .data import SCRATCH_IGNORE, import_rows
378
+ from .data import import_rows
381
379
  from .mapping import FILE, Mapping, apply_mapping, load_mapping, parse_mapping, validate_mapping
382
380
  from .redaction import parse_mode
383
381
  from .transform import DEFAULT_TIMEOUT, check_rows, transform_records
@@ -448,20 +446,14 @@ def ingest_records(
448
446
  return report
449
447
  # The contract's own refusal of a row raises from the import, as `data check`
450
448
  # raises it: the CLI turns the bridge's rejection into one refusal naming the pointer.
451
- if dry_run:
452
- with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
453
- staged = Path(scratch) / "world"
454
- shutil.copytree(root, staged, ignore=shutil.ignore_patterns(*SCRATCH_IGNORE))
455
- proven = import_rows(staged, projected, mode=mode, redact=redact)
456
- report.update({"kind": proven["kind"], "entity_counts": proven["entity_counts"], "batch": proven.get("batch")})
457
- return report
458
- result = import_rows(root, projected, mode=mode, redact=redact)
449
+ result = import_rows(root, projected, mode=mode, redact=redact, dry_run=dry_run)
459
450
  report.update(
460
451
  {
461
452
  "kind": result["kind"],
462
453
  "entity_counts": result["entity_counts"],
463
454
  "batch": result.get("batch"),
464
455
  "changed": result["changed"],
456
+ **{key: result[key] for key in ("proof", "deferred") if key in result},
465
457
  }
466
458
  )
467
459
  return report
@@ -37,12 +37,13 @@ from pathlib import Path
37
37
  from typing import Any, Callable
38
38
 
39
39
  from . import snapshot as snapshots
40
- from .api import Router, WorldApp, load_api_settings, operation_request, send_response, vendor_credential, with_query
41
- from .host_surfaces import UiSurface, api_target, load_ui_surface, static_file
40
+ from .api import Router, WorldApp, load_api_settings, send_response, vendor_credential, with_query
42
41
  from .compiler import bundle_mode, fail
43
42
  from .conform import PIN
44
43
  from .connector import SPEC, load_spec
45
44
  from .dependencies import local_dependency_trees
45
+ from .host_surfaces import UiSurface, api_target, load_ui_surface, static_file
46
+ from .runtime import call_operation, client_grant, task_card
46
47
  from .tools_world import (
47
48
  DependencyWorld,
48
49
  ToolsTask,
@@ -52,30 +53,11 @@ from .tools_world import (
52
53
  load_tools_version,
53
54
  )
54
55
  from .validation import ContractError
56
+ from .verbs import refusal, session_verb, unknown_verb
55
57
 
56
58
  LOCAL_TASK = ToolsTask(id="local", prompt="A local session; every verifier grades.")
57
59
 
58
60
 
59
- def refusal(error: ContractError) -> dict:
60
- """A refused call as the answer a hosted session gives it: an unknown tool,
61
- arguments off the schema, a tool outside the task's allow-list, a write the
62
- store will not take, all ``{status: 400, body: {error, detail}}``."""
63
- return {"status": 400, "body": {"error": error.detail.get("code", "host"), "detail": str(error)}}
64
-
65
-
66
- def seed_args(body: dict) -> dict:
67
- """The seed a ``POST /__session/seed`` body asks for: ``{rows, mode}``, the
68
- same under ``args``, or bare ``{entity: [row, ...]}`` — the shape a rows
69
- file has — taken as the rows of an append."""
70
- if isinstance(body.get("args"), dict):
71
- body = body["args"]
72
- if "rows" in body or "mode" in body:
73
- return body
74
- if body and all(isinstance(value, list) for value in body.values()):
75
- return {"rows": body}
76
- return body
77
-
78
-
79
61
  def has_http(root: str | Path) -> bool:
80
62
  """Whether the world serves HTTP at all: only a tree with ``connector.toml``
81
63
  has routes. A schema world without one is tools-only, and that is a whole
@@ -344,40 +326,6 @@ SESSION_PREFIX = "/__session/"
344
326
  UI_PREFIX = "/__ui/"
345
327
 
346
328
 
347
- def tool_contracts(bundle: Path, prefix: str = "") -> dict[str, dict]:
348
- """Each compiled tool's ``{description, input}`` from the bundle's
349
- ``tool-schemas.json`` (refs inlined by the compiler), keyed by served name."""
350
- path = bundle / "tool-schemas.json"
351
- if not path.is_file():
352
- return {}
353
- tools = json.loads(path.read_text(encoding="utf-8")).get("tools") or {}
354
- return {
355
- f"{prefix}{name}": {"description": str(entry.get("description") or ""), "input": entry.get("input_schema")}
356
- for name, entry in tools.items()
357
- }
358
-
359
-
360
- def task_card(world: ToolsWorld) -> dict:
361
- """What an outside agent is handed about the session's task: ``{task,
362
- prompt, tools, contracts, verifiers, worlds}`` — ``tools`` the served names,
363
- ``contracts`` each one's description and input schema, ``verifiers`` the
364
- root's that grade, ``worlds`` each dependency alias to the verifiers of its
365
- own that grade (``slack/message_posted`` in the grade's ``rewards``)."""
366
- task = world.task
367
- contracts = tool_contracts(world.version.bundle)
368
- for alias, dep in world.dependencies.items():
369
- contracts.update(tool_contracts(dep.version.bundle, f"{alias}."))
370
- return {
371
- "task": task.id,
372
- "prompt": task.prompt,
373
- "tools": list(world.served),
374
- "contracts": {name: contracts[name] for name in world.served if name in contracts},
375
- "verifiers": task.selected(world.version.verifiers),
376
- "worlds": {alias: task.selected_of(alias, dep.version.verifiers) for alias, dep in world.dependencies.items()},
377
- "metadata": dict(task.metadata),
378
- }
379
-
380
-
381
329
  def make_session_handler(
382
330
  world: ToolsWorld,
383
331
  app: WorldApp | None,
@@ -403,21 +351,33 @@ def make_session_handler(
403
351
  # The page's calls and a `/__session/call` of an operation are presented
404
352
  # as a granted client, as the host presents them; the world's own token
405
353
  # dance stays available beside.
406
- grant = local_grant(app, api_key)
354
+ grant = client_grant(app, api_key)
407
355
 
408
- def close(_body: dict) -> dict:
356
+ def close() -> dict:
409
357
  closing.set()
410
358
  return {"closed": True, "task": world.task.id}
411
359
 
412
- verbs: dict[tuple[str, str], Callable[[dict], Any]] = {
413
- ("GET", "tools"): lambda _body: {"tools": list(world.served)},
414
- ("GET", "task"): lambda _body: task_card(world),
415
- ("GET", "state"): lambda _body: world.state(),
416
- ("POST", "grade"): lambda _body: world.grade(),
417
- ("POST", "reset"): lambda _body: world.reset(),
418
- ("POST", "seed"): lambda body: world.seed(seed_args(body)),
419
- ("POST", "call"): lambda body: _call(world, app, body, api_key=api_key, grant=grant),
420
- ("POST", "close"): close,
360
+ def seed(rows: dict, mode: str, redact: str) -> dict:
361
+ if redact != "off":
362
+ fail("host", "a schema world has no [redact] policy; seed it with redact=off")
363
+ return world.seed({"rows": rows, "mode": mode})
364
+
365
+ def grade(report: str | None = None) -> dict:
366
+ # The local session grades by the world's own verifiers; an agent's
367
+ # report is read by a session task's grader, which only the host runs.
368
+ return world.grade()
369
+
370
+ # The session protocol (:data:`verbs.SESSION_VERBS`) bound to this
371
+ # session's operations, by route; the host binds the same rows to its own.
372
+ operations: dict[str, Callable[..., dict]] = {
373
+ "GET tools": lambda: {"tools": list(world.served)},
374
+ "GET task": lambda: task_card(world),
375
+ "GET state": world.state,
376
+ "POST grade": grade,
377
+ "POST reset": world.reset,
378
+ "POST seed": seed,
379
+ "POST call": lambda tool, args: call_served(world, app, tool, args, api_key=api_key, grant=grant),
380
+ "POST close": close,
421
381
  }
422
382
 
423
383
  class Handler(BaseHTTPRequestHandler):
@@ -498,14 +458,11 @@ def make_session_handler(
498
458
  return self._reply(400, {"error": "malformed json", "detail": str(error)})
499
459
  if not isinstance(body, dict):
500
460
  return self._reply(400, {"error": "malformed json", "detail": "the body must be a json object"})
501
- handler = verbs.get((method, verb))
502
- if handler is None:
503
- known = ", ".join(f"{m} {SESSION_PREFIX}{v}" for m, v in verbs)
504
- return self._reply(
505
- 404, {"error": f"unknown session route {method} {SESSION_PREFIX}{verb}; one of {known}"}
506
- )
461
+ row = session_verb(method, verb or None)
462
+ if row is None or row.hosted:
463
+ return self._reply(404, unknown_verb(method, verb or None, hosted=False, prefix=SESSION_PREFIX))
507
464
  try:
508
- return self._reply(200, handler(body))
465
+ return self._reply(200, operations[row.route](**row.read(body)))
509
466
  except ContractError as error:
510
467
  return self._reply(**refusal(error))
511
468
 
@@ -541,16 +498,6 @@ def _local_origin(origin: str) -> bool:
541
498
  return host in ("127.0.0.1", "localhost", "::1")
542
499
 
543
500
 
544
- def local_grant(app: WorldApp | None, api_key: str | None) -> str | None:
545
- """The bearer an ``oauth2`` world's routes take when the session itself
546
- calls them (the page's api prefix, an operation by name); None for any
547
- other scheme, or when the world refuses the client-credentials grant."""
548
- if app is None or app.router.spec.scheme != "oauth2":
549
- return None
550
- issued = app.router.issue_token({"grant_type": "client_credentials", "client_secret": api_key or ""}, api_key)
551
- return issued["access_token"] if issued else None
552
-
553
-
554
501
  def call_served(
555
502
  world: ToolsWorld,
556
503
  app: WorldApp | None,
@@ -566,51 +513,10 @@ def call_served(
566
513
  same store, ``{status, body}`` the route's own. A name that is both is the
567
514
  tool's."""
568
515
  if app is not None and tool not in world.version.tool_names and tool in app.router.spec.operations:
569
- return call_operation(world, app, tool, args, api_key=api_key, grant=grant)
516
+ return call_operation(app, tool, args, api_key or "", allowed=world.allowed, grant=grant)
570
517
  return world.call(tool, args)
571
518
 
572
519
 
573
- def call_operation(
574
- world: ToolsWorld,
575
- app: WorldApp,
576
- name: str,
577
- args: dict,
578
- *,
579
- api_key: str | None = PIN,
580
- grant: str | None = None,
581
- ) -> dict:
582
- """An operation of connector.toml called by name (:func:`operation_request`
583
- reads the args); the world's own credential is presented, as the host
584
- presents a session's. The answer is the route's ``{status, body}``."""
585
- operation = app.router.spec.operations.get(name)
586
- if operation is None:
587
- known = ", ".join(sorted(app.router.spec.operations)) or "(none)"
588
- fail("host", f"unknown tool '{name}'; this world serves: {known}")
589
- if world.allowed is not None and name not in world.allowed:
590
- fail(
591
- "tool_not_allowed",
592
- f"tool '{name}' is not allowed by this session's task; it allows: {', '.join(sorted(world.allowed))}",
593
- )
594
- method, target, raw = operation_request(app.router.spec, operation, args)
595
- headers, credential_query = vendor_credential(app.router.spec, api_key or "", grant=grant)
596
- status, answer, _ = app.handle(
597
- method, with_query(target, credential_query), {"content-type": "application/json", **headers}, raw, "127.0.0.1"
598
- )
599
- return {"status": status, "body": answer}
600
-
601
-
602
- def _call(
603
- world: ToolsWorld, app: WorldApp | None, body: dict, *, api_key: str | None = PIN, grant: str | None = None
604
- ) -> dict:
605
- tool = body.get("tool")
606
- if not isinstance(tool, str) or not tool:
607
- fail("host", "tool must be a tool name (root) or alias.tool (a dependency's)")
608
- args = body.get("args") or {}
609
- if not isinstance(args, dict):
610
- fail("host", "args must be an object of the tool's arguments")
611
- return call_served(world, app, tool, args, api_key=api_key, grant=grant)
612
-
613
-
614
520
  def _apply_scenario(world: ToolsWorld, root: Path, name: str) -> None:
615
521
  scenarios = snapshots.scenario_dirs(root)
616
522
  if name not in scenarios:
@@ -51,12 +51,12 @@ from __future__ import annotations
51
51
  import hashlib
52
52
  import json
53
53
  import re
54
- import tomllib
55
54
  from dataclasses import dataclass
56
55
  from datetime import datetime, timezone
57
56
  from pathlib import Path
58
57
  from typing import Any, Iterable
59
58
 
59
+ from .._toml import tomllib
60
60
  from .transforms import apply_transforms, parse_transform
61
61
  from .validation import ContractError, fail
62
62
 
@@ -22,6 +22,7 @@ from pathlib import Path, PurePosixPath
22
22
  from typing import TYPE_CHECKING, Any
23
23
 
24
24
  from .._materialize import materialize_template
25
+ from ..world import WorldTree
25
26
  from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle
26
27
  from . import snapshot as snapshots
27
28
  from .compiler import retarget_empty_journals
@@ -325,7 +326,7 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
325
326
  "domain": contract.target,
326
327
  "package": contract.metadata["package"],
327
328
  "initial_state": initial_state,
328
- "handler_execution": _handler_execution(bundle, execute),
329
+ "handler_execution": _handler_execution(bundle, execute, declared),
329
330
  "entity_counts": counts,
330
331
  "tools": sorted(declared),
331
332
  "verifiers": verifiers,
@@ -526,10 +527,14 @@ def may_execute() -> bool:
526
527
  return os.environ.get(EXECUTE_ENV) == "1"
527
528
 
528
529
 
529
- def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
530
+ def _handler_execution(bundle: Path, execute: bool, declared: set[str]) -> dict[str, Any]:
530
531
  """Whether the handlers were run: imported, every declared tool bound, the
531
532
  ``[dependencies]`` opened from their paths — one session of the world,
532
- opened and closed. Only where authored code may run."""
533
+ opened and closed. Only where authored code may run. ``tools`` is the
534
+ declared handlers the session bound (the report's top-level ``tools``);
535
+ ``served`` every name ``worlds call`` answers there — handlers, the connector
536
+ routes as operations, a dependency's ``alias.tool`` — so the two lists differ
537
+ by design and say what each counts."""
533
538
  if not execute:
534
539
  return {
535
540
  "status": "not_run",
@@ -545,24 +550,24 @@ def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
545
550
  from .local import open_world
546
551
 
547
552
  with open_world(bundle, http=False) as session:
548
- return {"status": "ok", "tools": session.tools}
549
-
550
-
551
- API_BUNDLE = "gateway-api-world.toml"
552
- BUNDLED_WORLD = "server/world"
553
+ served = list(session.tools)
554
+ unbound = sorted(declared - set(served))
555
+ if unbound:
556
+ fail(
557
+ "handler",
558
+ "Declared in schema/world.json (tools) but not bound when the session opened: " + ", ".join(unbound),
559
+ "/tools",
560
+ )
561
+ return {"status": "ok", "tools": sorted(declared), "served": served}
553
562
 
554
563
 
555
564
  def world_root(root: Path) -> Path:
556
- """The world inside the tree the platform holds. ``bench push`` uploads a schema
557
- world as its api bundle, so a version's tree is ``gateway-api-world.toml`` +
558
- ``seed/`` + ``scenarios/`` + ``server/world/<the authoring world>``: that nested
559
- world is what data commands operate on. A plain tree is its own root."""
560
- nested = root / BUNDLED_WORLD
561
- if (root / API_BUNDLE).is_file() and (
562
- (nested / "connector.toml").is_file() or (nested / "schema/world.json").is_file()
563
- ):
564
- return nested
565
- return root
565
+ """The world inside the tree the platform holds (``WorldTree.world``).
566
+ ``bench push`` uploads a schema world as its api bundle, so a version's
567
+ tree is ``gateway-api-world.toml`` + ``seed/`` + ``scenarios/`` +
568
+ ``server/world/<the authoring world>``: that nested world is what data
569
+ commands operate on. A plain tree is its own root."""
570
+ return WorldTree.from_dir(root).world
566
571
 
567
572
 
568
573
  def _write_files(root: Path, entries: list) -> None:
@@ -968,12 +973,17 @@ def _import_data(request: dict) -> dict:
968
973
  if merge not in MERGES:
969
974
  fail("input", f"merge must be one of {', '.join(MERGES)}")
970
975
  redact = parse_mode(request.get("redact"))
976
+ dry_run = _flag(request, "dry_run", False)
971
977
  with tempfile.TemporaryDirectory(prefix="gateway-import-") as directory:
972
978
  root = Path(directory) / "bundle"
973
979
  root.mkdir()
974
980
  _write_files(root, request.get("files"))
975
981
  world = world_root(root)
976
- result = import_rows(world, rows, mode=mode, merge=merge, redact=redact)
982
+ result = import_rows(world, rows, mode=mode, merge=merge, redact=redact, dry_run=dry_run)
983
+ keys = ("kind", "entity_counts", "changed")
984
+ report = {"status": "ok", **{key: result[key] for key in keys}}
985
+ if dry_run:
986
+ return {**report, **{key: result[key] for key in ("proof", "deferred") if key in result}}
977
987
  if result["kind"] == KIND_SCHEMA and snapshots.default_snapshot(world).is_file():
978
988
  # The journals grew; a snapshot built before this import would be
979
989
  # served over them. Rebuild so the tree that travels back is one thing.
@@ -986,13 +996,7 @@ def _import_data(request: dict) -> dict:
986
996
  from .api import write_base_seed
987
997
 
988
998
  write_base_seed(world, root)
989
- return {
990
- "status": "ok",
991
- "kind": result["kind"],
992
- "entity_counts": result["entity_counts"],
993
- "changed": result["changed"],
994
- "files": _tree(root),
995
- }
999
+ return {**report, "files": _tree(root)}
996
1000
 
997
1001
 
998
1002
  def _ingest_calls(request: dict) -> dict:
@@ -0,0 +1,237 @@
1
+ """Proving rows against a world without writing them: the dry run behind
2
+ ``data check``, ``data import --dry-run``, ``ingest --dry-run`` and the bridge.
3
+
4
+ What the rows are proven *against* depends on where the world's data lives, and
5
+ :meth:`RowProof.of` picks the one proof that tells the truth about it:
6
+
7
+ * :class:`FilesProof` — an authored tree (no ``db/snapshot.json``): the data files
8
+ are the data, so the rows are imported into a scratch copy of the tree through
9
+ the same gate the real import runs;
10
+ * :class:`SnapshotProof` — a checkout pinned to a hosted snapshot that carries the
11
+ snapshot itself (``db/state.sqlite`` current against the pin): the platform's
12
+ batch lane runs over a private copy of those bytes, so references, unique keys
13
+ and merges are checked against the rows the platform holds;
14
+ * :class:`ContractProof` — a pinned checkout without its snapshot: only what needs
15
+ no state is proven here (shape, keys, redaction); references and unique keys are
16
+ the platform batch's to decide, and the report says so under ``deferred``.
17
+
18
+ Dropping the pin and proving against the data files would be wrong both ways: a
19
+ row referencing a hosted key the files lack would be refused, and a row colliding
20
+ with a hosted unique key would pass.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import shutil
26
+ import tempfile
27
+ from contextlib import contextmanager
28
+ from pathlib import Path
29
+ from typing import Any, Iterable, Iterator
30
+
31
+ from .batch import ChunkSpec, build_batch, chunk_rows, validate_chunks
32
+ from .compiler import compile_bundle
33
+ from .validation import fail
34
+
35
+ Rows = dict[str, list[dict]]
36
+ #: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
37
+ SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
38
+ #: What a state-independent proof leaves to the platform batch.
39
+ DEFERRED = ("references", "unique")
40
+
41
+
42
+ @contextmanager
43
+ def proving_copy(root: Path) -> Iterator[Path]:
44
+ """A scratch copy of the tree to prove rows against and throw away: build
45
+ outputs and captures left behind."""
46
+ with tempfile.TemporaryDirectory(prefix="gateway-prove-") as scratch:
47
+ staged = Path(scratch) / "world"
48
+ shutil.copytree(root, staged, ignore=shutil.ignore_patterns(*SCRATCH_IGNORE))
49
+ yield staged
50
+
51
+
52
+ class RowProof:
53
+ """One dry run over one tree. ``check`` is the batch lane's report (rows,
54
+ refusals, entity counts) over chunk files; ``run`` is the import door's report
55
+ over in-memory rows, refusing with a ``ContractError`` as the import would."""
56
+
57
+ proof: str
58
+
59
+ def __init__(self, root: Path):
60
+ self.root = root
61
+
62
+ @classmethod
63
+ def of(cls, root: str | Path) -> "RowProof":
64
+ from .snapshot import current_default, pinned_default
65
+
66
+ root = Path(root)
67
+ pin = pinned_default(root)
68
+ if pin is None:
69
+ return FilesProof(root)
70
+ contract = compile_bundle(root, snapshots=False)
71
+ if current_default(root, contract) is not None:
72
+ return SnapshotProof(root, pin, contract)
73
+ return ContractProof(root, pin, contract)
74
+
75
+ def check(
76
+ self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
77
+ ) -> dict:
78
+ raise NotImplementedError
79
+
80
+ def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
81
+ raise NotImplementedError
82
+
83
+ def _report(self, checked: dict) -> dict:
84
+ return {**checked, "proof": self.proof}
85
+
86
+
87
+ class FilesProof(RowProof):
88
+ proof = "files"
89
+
90
+ def check(
91
+ self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
92
+ ) -> dict:
93
+ from .snapshot import EMPTY_DIGEST, build_default, default_snapshot
94
+
95
+ with proving_copy(self.root) as staged, tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
96
+ contract = compile_bundle(staged, snapshots=False)
97
+ build_default(staged, contract)
98
+ parent = Path(scratch) / "state.sqlite"
99
+ shutil.copyfile(default_snapshot(staged), parent)
100
+ return self._report(_batch(contract, parent, chunks, mode, merge, atomic, "check", EMPTY_DIGEST, redactor))
101
+
102
+ def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
103
+ from .data import import_rows
104
+
105
+ with proving_copy(self.root) as staged:
106
+ proven = import_rows(staged, rows, mode=mode, merge=merge, batch_id=batch_id, redact=redact)
107
+ return self._report({**proven, "changed": False})
108
+
109
+
110
+ class _PinnedProof(RowProof):
111
+ def __init__(self, root: Path, pin: dict, contract: Any):
112
+ super().__init__(root)
113
+ self.pin = pin
114
+ self.contract = contract
115
+
116
+ def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
117
+ from .data import KIND_SCHEMA, _merged, _refuse_shapes
118
+ from .redaction import parse_mode, redact_rows, require_policy
119
+
120
+ redact = parse_mode(redact)
121
+ redacted = redact_rows(require_policy(self.root, redact), rows, redact)
122
+ settled = _merged(self.root, redacted, merge)
123
+ _refuse_shapes(self.root, settled)
124
+ with tempfile.TemporaryDirectory(prefix="gateway-prove-") as scratch:
125
+ chunks = [
126
+ ChunkSpec(chunk.ordinal, chunk.path, chunk.entity)
127
+ for chunk in chunk_rows(_tagged(settled), Path(scratch) / "chunks")
128
+ ]
129
+ checked = self.check(chunks, mode=mode, merge=merge) if chunks else _empty()
130
+ _refuse(checked["refusals"], settled)
131
+ report = {"kind": KIND_SCHEMA, "batch": batch_id, "entity_counts": checked["entityCounts"], "changed": False}
132
+ if redact != "off":
133
+ report["redact"] = redact
134
+ return self._report({**report, **({"deferred": checked["deferred"]} if "deferred" in checked else {})})
135
+
136
+
137
+ class SnapshotProof(_PinnedProof):
138
+ proof = "snapshot"
139
+
140
+ def check(
141
+ self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
142
+ ) -> dict:
143
+ from .snapshot import default_snapshot
144
+
145
+ with tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
146
+ parent = Path(scratch) / "state.sqlite"
147
+ shutil.copyfile(default_snapshot(self.root), parent)
148
+ return self._report(
149
+ _batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin["batchDigest"], redactor)
150
+ )
151
+
152
+
153
+ class ContractProof(_PinnedProof):
154
+ proof = "contract"
155
+
156
+ def check(
157
+ self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
158
+ ) -> dict:
159
+ shaped = validate_chunks(self.contract, chunks, redactor=redactor)
160
+ counts = self.pin.get("entityCounts")
161
+ return self._report(
162
+ {
163
+ "rows": shaped["rows"],
164
+ "refusals": shaped["refusals"],
165
+ "refusalCount": len(shaped["refusals"]),
166
+ "entityCounts": dict(counts) if isinstance(counts, dict) else {},
167
+ "deferred": list(DEFERRED),
168
+ }
169
+ )
170
+
171
+
172
+ def _batch(contract, parent, chunks, mode, merge, atomic, batch_id, parent_digest, redactor) -> dict:
173
+ result = build_batch(
174
+ contract,
175
+ parent,
176
+ chunks,
177
+ mode=mode,
178
+ merge=merge,
179
+ atomic=atomic,
180
+ batch_id=batch_id,
181
+ parent_digest=parent_digest,
182
+ dry_run=True,
183
+ redactor=redactor,
184
+ )
185
+ return {
186
+ "rows": sum(1 for chunk in chunks for _ in _chunk_rows(chunk)),
187
+ "refusals": result["refusals"],
188
+ "refusalCount": result["refusalCount"],
189
+ "entityCounts": result["entityCounts"],
190
+ }
191
+
192
+
193
+ def _chunk_rows(chunk: ChunkSpec) -> Iterable[tuple[str, dict]]:
194
+ from .batch import iter_chunk_rows
195
+
196
+ return iter_chunk_rows(chunk.path, chunk.entity)
197
+
198
+
199
+ def _tagged(rows: Rows) -> Iterator[tuple[str, dict]]:
200
+ for entity, listed in rows.items():
201
+ for row in listed:
202
+ yield entity, row
203
+
204
+
205
+ def _empty() -> dict:
206
+ return {"rows": 0, "refusals": [], "refusalCount": 0, "entityCounts": {}}
207
+
208
+
209
+ def _refuse(refusals: list[dict], rows: Rows) -> None:
210
+ """The state the rows collide with, as one ``ContractError`` carrying every
211
+ refusal (shape refusals were raised earlier, with their own codes)."""
212
+ if not refusals:
213
+ return
214
+ first = refusals[0]
215
+ total = sum(len(listed) for listed in rows.values())
216
+ summary = f"{len(refusals)} of {total} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
217
+ fail(
218
+ "state",
219
+ summary + first["detail"],
220
+ first["pointer"],
221
+ entity=first["entity"],
222
+ index=first["index"],
223
+ refused=len(refusals),
224
+ rows=total,
225
+ refusals=refusals,
226
+ )
227
+
228
+
229
+ __all__ = [
230
+ "DEFERRED",
231
+ "SCRATCH_IGNORE",
232
+ "ContractProof",
233
+ "FilesProof",
234
+ "RowProof",
235
+ "SnapshotProof",
236
+ "proving_copy",
237
+ ]
@@ -31,11 +31,11 @@ from __future__ import annotations
31
31
 
32
32
  import json
33
33
  import re
34
- import tomllib
35
34
  from dataclasses import dataclass, field
36
35
  from pathlib import Path
37
36
  from typing import Any, Callable
38
37
 
38
+ from .._toml import tomllib
39
39
  from .validation import ContractError, fail
40
40
 
41
41
  REDACT_MODES = ("off", "apply", "refuse")