@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -37,7 +37,7 @@ the **pin** (:class:`SnapshotPin`): the hosted snapshot's ``batchDigest`` (and s
37
37
  fields), which replaces the data files as the default batch. The tree's ``data/*.json``
38
38
  then no longer describes the world's data — only that snapshot does — so a pinned tree
39
39
  is never rebuilt from, nor served from, its data files; the platform hands the file
40
- down (``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``). Where the file is not
40
+ down (``gateway worlds sandbox open``, the pod's ``WORLD_SNAPSHOT_*``). Where the file is not
41
41
  handed down (a sandbox), the pin itself proves which domain the hosted rows were stored
42
42
  under, so the contract still compiles and checks.
43
43
 
@@ -49,15 +49,18 @@ from __future__ import annotations
49
49
  import gzip
50
50
  import hashlib
51
51
  import json
52
+ import os
53
+ import re
52
54
  import shutil
53
55
  import urllib.request
56
+ import uuid
54
57
  from dataclasses import dataclass
55
58
  from datetime import datetime, timezone
56
59
  from pathlib import Path
57
- from typing import Any, Iterable
60
+ from typing import Any, Callable, Iterable
58
61
 
59
62
  from .._toml import tomllib
60
- from .compiler import LOCK, Contract, StorageContract, check_bundle
63
+ from .compiler import LOCK, Contract, check_bundle
61
64
  from .database import WorldDatabase
62
65
  from .domain_migration import DomainMigration
63
66
  from .validation import ContractError, canonical, fail, read_json
@@ -72,10 +75,15 @@ SIDECAR_KEYS = ("contractHash", "batchDigest", "entityCounts", "builtAt", "runti
72
75
  ENV_URL = "WORLD_SNAPSHOT_URL"
73
76
  ENV_SHA256 = "WORLD_SNAPSHOT_SHA256"
74
77
  ENV_DIGEST = "WORLD_SNAPSHOT_DIGEST"
78
+ #: Where a sandbox keeps the hosted snapshots its trees pin, outside every tree (``<root>/<batchDigest>/``).
79
+ STORE_ENV = "GATEWAY_SNAPSHOT_STORE"
80
+ _BATCH_DIGEST = re.compile(r"[0-9a-f]{64}")
75
81
  PIN_POINTER = f"/{DEFAULT_DIR}/{PIN}"
76
82
  #: The codes a pin is refused with: never a data refusal, so never discarded as one.
77
83
  PIN_REFUSALS = ("snapshot_pin", "snapshot_pinned")
78
84
  REIMPORT = f"remove {DEFAULT_DIR}/{PIN} and import every row again (`gateway worlds data import <slug> <rows>`)"
85
+ REBUILD_REMEDY = "recompile to rebuild it from the data files, or restore the domain it was built under"
86
+ PINNED_REMEDY = "recompile to carry it to this contract, or restore the domain it was built under"
79
87
  _BLOCK = 1024 * 1024
80
88
 
81
89
 
@@ -131,15 +139,15 @@ def pin_path(world: str | Path) -> Path:
131
139
 
132
140
 
133
141
  def refuse_unheld_pin(world: str | Path, contract: Contract) -> None:
134
- """A pinned tree that does not hold its hosted snapshot has no rows here: nothing
135
- is built, copied or served from its data files in their place."""
142
+ """A pinned tree that does not hold its hosted snapshot, and finds no copy of it in the
143
+ snapshot store, has no rows here: nothing is built, copied or served from its data files
144
+ in their place."""
136
145
  pin = SnapshotPin.of(world)
137
146
  if pin is not None:
138
147
  fail(
139
148
  "snapshot_pinned",
140
- f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold: "
141
- f"run the world on the platform (`gateway worlds session open <slug>`, load_world_to_sandbox), "
142
- f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files",
149
+ f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold"
150
+ + SnapshotStore.configured().remedy(pin),
143
151
  f"/{DEFAULT_DIR}/{snapshot_name(contract)}",
144
152
  )
145
153
 
@@ -185,6 +193,14 @@ class SnapshotPin:
185
193
  def batch_digest(self) -> str:
186
194
  return self.fields["batchDigest"]
187
195
 
196
+ def held(self, contract: Contract) -> bool:
197
+ """Whether the tree holds the pinned snapshot (the platform laid it beside the tree): its default
198
+ snapshot names this batch in the bytes its sidecar names. A pinned tree that does not hold it has
199
+ no rows of its own, so any snapshot in it was built from the data files the pin replaced."""
200
+ snapshot = default_snapshot(self.world, contract)
201
+ sidecar = _holding(snapshot, self.batch_digest)
202
+ return sidecar is not None and sidecar["fileSha256"] == file_sha256(snapshot)
203
+
188
204
  def counts(self, contract: Contract) -> dict[str, int]:
189
205
  """The hosted snapshot's entity counts as the platform recorded them, per entity."""
190
206
  counts = self.fields.get("entityCounts")
@@ -278,6 +294,86 @@ class SnapshotPin:
278
294
  return None
279
295
 
280
296
 
297
+ @dataclass(frozen=True)
298
+ class SnapshotStore:
299
+ """The hosted snapshots a sandbox keeps beside its trees, never in one (``$GATEWAY_SNAPSHOT_STORE``,
300
+ ``<root>/<batchDigest>/<snapshot name>`` with its sidecar), where the platform lays the one a
301
+ sandbox's version serves. A pinned tree that holds no snapshot of its own reads its rows from the
302
+ copy of its batch; ``root`` is None where no store is configured or its directory does not exist
303
+ (a sandbox the platform lays no snapshot into)."""
304
+
305
+ root: Path | None
306
+
307
+ @classmethod
308
+ def configured(cls) -> "SnapshotStore":
309
+ root = os.environ.get(STORE_ENV)
310
+ return cls(Path(root) if root and Path(root).is_dir() else None)
311
+
312
+ def held(self, pin: SnapshotPin, contract: Contract) -> tuple[Path, dict] | None:
313
+ """The copy of the pinned batch and its sidecar, opened under ``contract``: the copy itself when
314
+ its rows were stored under this contract's storage, else the copy carried to it (kept beside the
315
+ original) when the contract only adds to their domain. None without a copy that is the bytes its
316
+ sidecar names and, when the pin names them, the pin's own bytes."""
317
+ if self.root is None or not _BATCH_DIGEST.fullmatch(pin.batch_digest):
318
+ return None
319
+ copy = self.root / pin.batch_digest / snapshot_name(contract)
320
+ sidecar = _readable(lambda: _holding(copy, pin.batch_digest))
321
+ if sidecar is None or sidecar["fileSha256"] != file_sha256(copy):
322
+ return None
323
+ if sidecar["fileSha256"] != pin.fields.get("fileSha256", sidecar["fileSha256"]):
324
+ return None
325
+ if storage_refusal(copy, contract) is None:
326
+ return copy, sidecar
327
+ return self._carried(copy, pin, contract)
328
+
329
+ def remedy(self, pin: SnapshotPin) -> str:
330
+ """How a refusal of a pinned snapshot neither the tree nor this store holds ends."""
331
+ if self.root is None:
332
+ return (
333
+ ": run the world on the platform (`gateway worlds session open <slug>`, `gateway worlds sandbox open <slug>`), "
334
+ f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files"
335
+ )
336
+ return (
337
+ f", nor does this sandbox hold a whole copy at {self.root / pin.batch_digest}: the platform lays there, "
338
+ "before each `gateway worlds sandbox exec`, the snapshot this sandbox's version serves (that exec's "
339
+ f"warnings say why it could not). Restore {DEFAULT_DIR}/{PIN} as the version has it if it was edited, or "
340
+ "remove that directory and exec again; meanwhile `gateway worlds data sql <slug>` reads the rows from your "
341
+ "machine"
342
+ )
343
+
344
+ def _carried(self, copy: Path, pin: SnapshotPin, contract: Contract) -> tuple[Path, dict]:
345
+ carried = copy.parent / contract_hash(contract) / copy.name
346
+ kept = _readable(lambda: holds(carried, contract, pin.batch_digest))
347
+ if kept is not None:
348
+ return carried, kept
349
+ for other in copy.parent.iterdir():
350
+ if other.is_dir() and other.name != carried.parent.name:
351
+ shutil.rmtree(other, ignore_errors=True)
352
+ fresh = carried.with_name(f".{carried.name}.{uuid.uuid4().hex}")
353
+ fresh.parent.mkdir(parents=True, exist_ok=True)
354
+ WorldDatabase.clone_file(copy, fresh)
355
+ try:
356
+ DomainMigration.carry(fresh, contract, pin.world)
357
+ with WorldDatabase.open(fresh, contract, audit=False) as store:
358
+ counts = store.counts()
359
+ except BaseException:
360
+ fresh.unlink(missing_ok=True)
361
+ raise
362
+ _replace(carried, fresh)
363
+ return carried, write_sidecar(carried, contract_hash(contract), pin.batch_digest, counts)
364
+
365
+
366
+ def _readable(read: Callable[[], dict | None]) -> dict | None:
367
+ """``read()``, or None when the sidecar it reads cannot be read: a store copy is the platform's to lay
368
+ again and a carried copy the store's to carry again, so an unreadable sidecar there is a missing copy."""
369
+ try:
370
+ return read()
371
+ except ContractError as error:
372
+ if error.detail.get("code") != "snapshot":
373
+ raise
374
+ return None
375
+
376
+
281
377
  def expected_default_digest(world: str | Path) -> str:
282
378
  """The batch digest the default snapshot must carry: the pin's for a data-only
283
379
  version, the data files' for an authored tree."""
@@ -409,9 +505,9 @@ def storage_refusal(snapshot: Path, contract: Contract) -> ContractError | None:
409
505
  return error
410
506
 
411
507
 
412
- def verify(snapshot: Path, contract: Contract) -> dict:
508
+ def verify(snapshot: Path, contract: Contract, remedy: str = REBUILD_REMEDY) -> dict:
413
509
  """The sidecar of a snapshot whose bytes it names and whose rows ``contract`` may
414
- open: anything else is refused."""
510
+ open: anything else is refused, saying what a recompile does about it."""
415
511
  shown = _shown(snapshot)
416
512
  sidecar = read_sidecar(snapshot)
417
513
  if sidecar is None:
@@ -424,8 +520,7 @@ def verify(snapshot: Path, contract: Contract) -> dict:
424
520
  if refusal is not None:
425
521
  fail(
426
522
  "snapshot",
427
- f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); "
428
- "recompile to rebuild it from the data files, or restore the domain it was built under",
523
+ f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); {remedy}",
429
524
  shown,
430
525
  changed=refusal.detail.get("changed", []),
431
526
  )
@@ -523,9 +618,14 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
523
618
  def build_all(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
524
619
  """The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
525
620
  A pinned tree that does not hold its hosted snapshot builds none (its data lives on the
526
- platform): the default is the pin's view, once the pin proves the rows' domain."""
621
+ platform) and keeps none: the snapshots its data files built claim data that no longer
622
+ applies. The default is the pin's view, once the pin proves the rows' domain."""
527
623
  world = Path(world)
528
624
  pin = SnapshotPin.of(world)
625
+ if pin is not None and not pin.held(contract):
626
+ view = pin.view(contract)
627
+ discard_all(world)
628
+ return {"default": view, "scenarios": {}}
529
629
  if pin is not None:
530
630
  carry_held(world, contract)
531
631
  if pin is not None and holds(default_snapshot(world, contract), contract, pin.batch_digest) is None:
@@ -572,23 +672,24 @@ def discard_all(world: str | Path) -> None:
572
672
 
573
673
 
574
674
  def check_all(world: str | Path, contract: Contract) -> dict:
575
- """Every snapshot present in the tree, verified against its sidecar; absent ones are not required."""
675
+ """Every snapshot present in the tree, verified against its sidecar; absent ones are not required.
676
+ A pinned tree that does not hold its hosted snapshot has none to verify: nothing opens the
677
+ snapshots its data files built (compile removes them), so they are never refused."""
576
678
  world = Path(world)
577
679
  found: dict[str, Any] = {"default": None, "scenarios": {}}
680
+ pin = SnapshotPin.of(world)
681
+ if pin is not None and not pin.held(contract):
682
+ return found
683
+ remedy = PINNED_REMEDY if pin is not None else REBUILD_REMEDY
578
684
  snapshot = default_snapshot(world, contract)
579
685
  if snapshot.is_file() or sidecar_path(snapshot).is_file():
580
- found["default"] = verify(snapshot, contract)
581
- # A snapshot built from earlier data than the tree holds is reported, not
582
- # refused: authoring edits journals and opens the world, and a served world
583
- # replays the journals past a stale snapshot (WorldState opens only a
584
- # current one). `stale` is in every check report so it never passes unseen.
585
- found["default"]["stale"] = SnapshotPin.of(world) is None and found["default"][
586
- "batchDigest"
587
- ] != default_batch_digest(world)
686
+ found["default"] = verify(snapshot, contract, remedy)
687
+ # Reported, not refused: a served world replays the journals past a stale snapshot.
688
+ found["default"]["stale"] = pin is None and found["default"]["batchDigest"] != default_batch_digest(world)
588
689
  for name in scenario_dirs(world):
589
690
  snapshot = scenario_snapshot(world, name, contract)
590
691
  if snapshot.is_file() or sidecar_path(snapshot).is_file():
591
- found["scenarios"][name] = verify(snapshot, contract)
692
+ found["scenarios"][name] = verify(snapshot, contract, remedy)
592
693
  return found
593
694
 
594
695
 
@@ -623,24 +724,40 @@ def current_default(world: str | Path, contract: Contract) -> dict | None:
623
724
  return holds(snapshot, contract, expected_default_digest(world))
624
725
 
625
726
 
727
+ def held_default(world: str | Path, contract: Contract) -> tuple[Path, dict] | None:
728
+ """The default snapshot a reader opens, and its sidecar: the tree's own when it holds the tree's
729
+ data (or the pin's), else, for a pinned tree, the snapshot store's copy of the pinned batch."""
730
+ world = Path(world)
731
+ sidecar = current_default(world, contract)
732
+ if sidecar is not None:
733
+ return default_snapshot(world, contract), sidecar
734
+ pin = SnapshotPin.of(world)
735
+ if pin is None:
736
+ return None
737
+ return SnapshotStore.configured().held(pin, contract)
738
+
739
+
626
740
  @dataclass(frozen=True)
627
741
  class ServedSnapshot:
628
742
  """What a served world opens: its contract, the tree checked against it, and the
629
- sidecar of the default snapshot when that holds the tree's data (or the pin's) in
630
- rows the contract may open, None on the legacy path. A pinned tree that holds no
631
- snapshot is refused. Checking hashes the snapshot, so a version cache checks its
632
- immutable tree once and hands the answer to every session of the version."""
743
+ default snapshot (``path`` and its ``sidecar``) when one holds the tree's data (or the
744
+ pin's) in rows the contract may open, None on the legacy path. A pinned tree that holds
745
+ no snapshot, and finds none in the snapshot store, is refused. Checking hashes the
746
+ snapshot, so a version cache checks its immutable tree once and hands the answer to
747
+ every session of the version."""
633
748
 
634
749
  contract: Contract
635
750
  sidecar: dict | None
751
+ path: Path | None = None
636
752
 
637
753
  @classmethod
638
754
  def of(cls, world: str | Path) -> "ServedSnapshot":
639
755
  contract = check_bundle(world)
640
- sidecar = current_default(world, contract)
641
- if sidecar is None:
756
+ held = held_default(world, contract)
757
+ if held is None:
642
758
  refuse_unheld_pin(world, contract)
643
- return cls(contract, sidecar)
759
+ return cls(contract, None)
760
+ return cls(contract, held[1], held[0])
644
761
 
645
762
 
646
763
  def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest: str) -> dict:
@@ -699,6 +816,8 @@ __all__ = [
699
816
  "SIDECAR",
700
817
  "ServedSnapshot",
701
818
  "SnapshotPin",
819
+ "SnapshotStore",
820
+ "STORE_ENV",
702
821
  "batch_digest",
703
822
  "build_all",
704
823
  "build_default",
@@ -713,6 +832,7 @@ __all__ = [
713
832
  "expected_default_digest",
714
833
  "fetch",
715
834
  "file_sha256",
835
+ "held_default",
716
836
  "holds",
717
837
  "lock_hash",
718
838
  "migration_view",
@@ -0,0 +1,228 @@
1
+ """The files under the World Host's request ledger spill, as tenant code could have left them: version
2
+ workers share the host's user and cache, so every name is reached from a directory held open by
3
+ descriptor and never through a link, a file is written only when it is a regular one and opened without
4
+ waiting on a FIFO, a directory the host keeps is given back its owner's access before it is used and
5
+ again whenever tenant code took it, and every line carries a seal only the host can make
6
+ (:class:`SpillSeal`)."""
7
+
8
+ from __future__ import annotations
9
+
10
+ import errno
11
+ import hashlib
12
+ import hmac
13
+ import os
14
+ import shutil
15
+ import stat
16
+ import uuid
17
+ from collections.abc import Callable, Iterator
18
+ from contextlib import contextmanager
19
+ from functools import partial
20
+ from pathlib import Path
21
+ from typing import TypeVar
22
+
23
+ from .call_log import read_tail
24
+ from .handler_errors import log_line
25
+
26
+ TOMBSTONE = ".gone-"
27
+ DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK
28
+ WRITING = os.O_WRONLY | os.O_NOFOLLOW | os.O_NONBLOCK
29
+ Done = TypeVar("Done")
30
+
31
+
32
+ def _write_all(descriptor: int, data: bytes) -> None:
33
+ view = memoryview(data)
34
+ while view:
35
+ view = view[os.write(descriptor, view) :]
36
+
37
+
38
+ class SpillSeal:
39
+ """The seal on every spill line: an HMAC under a key derived from the control-plane secret, which a
40
+ version worker is never given, over the account's ``source``, the line's ``place`` (a segment line's
41
+ segment and ordinal, a record line's predecessor's tag) and the line, so a line written, changed,
42
+ copied or moved by anything but this host fails it."""
43
+
44
+ OVERHEAD = 66
45
+
46
+ def __init__(self, secret: str) -> None:
47
+ self._key = hmac.new(secret.encode("utf-8"), b"world-host-ledger-spill/2", hashlib.sha256).digest()
48
+
49
+ def seal(self, source: str, data: bytes, place: bytes = b"") -> bytes:
50
+ return self._tag(source, place, data) + b" " + data + b"\n"
51
+
52
+ def opened(self, source: str, line: bytes, place: bytes = b"") -> bytes | None:
53
+ tag, _, data = line.partition(b" ")
54
+ return data if hmac.compare_digest(tag, self._tag(source, place, data)) else None
55
+
56
+ @staticmethod
57
+ def tag(line: bytes) -> bytes:
58
+ return line.partition(b" ")[0]
59
+
60
+ def _tag(self, source: str, place: bytes, data: bytes) -> bytes:
61
+ message = source.encode("ascii") + b"\n" + place + b"\n" + data
62
+ return hmac.new(self._key, message, hashlib.sha256).hexdigest().encode("ascii")
63
+
64
+
65
+ class SpillDirectory:
66
+ """A directory held open by descriptor: every name in it is reached without following a link, every
67
+ file read as tenant code could have written it and written only as a regular file."""
68
+
69
+ def __init__(self, descriptor: int) -> None:
70
+ self.descriptor = descriptor
71
+
72
+ @classmethod
73
+ def opened(cls, name: str | Path, within: int | None = None) -> SpillDirectory:
74
+ return cls(os.open(name, DIRECTORY, dir_fd=within))
75
+
76
+ @classmethod
77
+ def owned(cls, name: str, within: int) -> SpillDirectory:
78
+ """``name`` opened and given its owner's access alone; one the host cannot make its own raises."""
79
+ directory = cls.opened(name, within)
80
+ try:
81
+ directory.own()
82
+ except OSError:
83
+ directory.close()
84
+ raise
85
+ return directory
86
+
87
+ def own(self) -> None:
88
+ """Its owner's access alone again, through the descriptor it is held by: tenant code can change
89
+ a held directory's mode, never which directory is held."""
90
+ os.fchmod(self.descriptor, 0o700)
91
+
92
+ def reowned(self, operation: Callable[[], Done]) -> Done:
93
+ """``operation`` on a name in this directory, run once more after the directory is made the host's
94
+ own again (:meth:`own`) when tenant code took the host's access to it, its write or search bit."""
95
+ try:
96
+ return operation()
97
+ except PermissionError:
98
+ self.own()
99
+ return operation()
100
+
101
+ def close(self) -> None:
102
+ os.close(self.descriptor)
103
+
104
+ @contextmanager
105
+ def child(self, name: str) -> Iterator[SpillDirectory]:
106
+ directory = self.reowned(partial(SpillDirectory.owned, name, self.descriptor))
107
+ try:
108
+ yield directory
109
+ finally:
110
+ directory.close()
111
+
112
+ def names(self) -> list[str]:
113
+ return sorted(os.listdir(self.descriptor))
114
+
115
+ def mode(self, name: str) -> int | None:
116
+ mode, _ = self._status(name)
117
+ return mode
118
+
119
+ def size(self, name: str) -> int | None:
120
+ mode, size = self._status(name)
121
+ return size if mode is not None and stat.S_ISREG(mode) else None
122
+
123
+ def ensure(self, name: str) -> None:
124
+ """A directory ``name`` here, made when missing (:meth:`child` refuses anything else at it)."""
125
+ try:
126
+ os.mkdir(name, 0o700, dir_fd=self.descriptor)
127
+ except FileExistsError:
128
+ return None
129
+
130
+ def make(self, name: str) -> None:
131
+ os.mkdir(name, 0o700, dir_fd=self.descriptor)
132
+
133
+ def create(self, name: str, data: bytes) -> None:
134
+ """``data`` as a new regular file ``name``; refused when anything, a link included, is there."""
135
+ descriptor = self._writable(name, os.O_CREAT | os.O_EXCL)
136
+ try:
137
+ _write_all(descriptor, data)
138
+ except OSError:
139
+ self.unlink(name)
140
+ raise
141
+ finally:
142
+ os.close(descriptor)
143
+
144
+ def add(self, name: str, data: bytes) -> None:
145
+ """``data`` appended to the regular file ``name``, made when missing."""
146
+ descriptor = self._writable(name, os.O_CREAT | os.O_APPEND)
147
+ try:
148
+ _write_all(descriptor, data)
149
+ finally:
150
+ os.close(descriptor)
151
+
152
+ def replace(self, name: str, data: bytes) -> None:
153
+ """``data`` as the whole of ``name`` at once: written beside it, then renamed over it."""
154
+ written = f".{name}-{uuid.uuid4().hex}"
155
+ self.create(written, data)
156
+ try:
157
+ os.rename(written, name, src_dir_fd=self.descriptor, dst_dir_fd=self.descriptor)
158
+ except OSError:
159
+ self.unlink(written)
160
+ raise
161
+
162
+ def tail(self, name: str, max_bytes: int) -> bytes | None:
163
+ """The newest ``max_bytes`` of ``name`` through :func:`~.call_log.read_tail` (anything but a regular
164
+ file reads as empty); None when it cannot be opened or read."""
165
+ try:
166
+ return read_tail(name, max_bytes, self.descriptor)
167
+ except OSError:
168
+ return None
169
+
170
+ def unlink(self, name: str) -> None:
171
+ try:
172
+ os.unlink(name, dir_fd=self.descriptor)
173
+ except FileNotFoundError:
174
+ return None
175
+
176
+ def discard(self, name: str) -> None:
177
+ """Move whatever is at ``name`` out of the way at once (a link as a link, a directory given its owner's
178
+ access first, which moving one needs on some kernels); :meth:`sweep` deletes it."""
179
+ if stat.S_ISDIR(self.mode(name) or 0):
180
+ self._own(name)
181
+ try:
182
+ os.rename(name, f"{TOMBSTONE}{uuid.uuid4().hex}", src_dir_fd=self.descriptor, dst_dir_fd=self.descriptor)
183
+ except FileNotFoundError:
184
+ return None
185
+
186
+ def sweep(self) -> None:
187
+ """Delete what :meth:`discard` set aside, as far as it can; what it cannot is logged and left."""
188
+ for name in (name for name in self.names() if name.startswith(TOMBSTONE)):
189
+ self._delete(name)
190
+
191
+ def _writable(self, name: str, flags: int) -> int:
192
+ descriptor = os.open(name, WRITING | flags, 0o600, dir_fd=self.descriptor)
193
+ try:
194
+ regular = stat.S_ISREG(os.fstat(descriptor).st_mode)
195
+ except OSError:
196
+ os.close(descriptor)
197
+ raise
198
+ if regular:
199
+ return descriptor
200
+ os.close(descriptor)
201
+ raise OSError(errno.EINVAL, "not a regular file", name)
202
+
203
+ def _own(self, name: str) -> None:
204
+ try:
205
+ os.chmod(name, 0o700, dir_fd=self.descriptor)
206
+ except OSError:
207
+ return None
208
+
209
+ def _delete(self, name: str) -> None:
210
+ mode, _ = self._status(name)
211
+ try:
212
+ self._remove(name, mode)
213
+ except OSError as error:
214
+ log_line(f"request ledger spill: {name} not removed: {type(error).__name__}")
215
+
216
+ def _remove(self, name: str, mode: int | None) -> None:
217
+ if mode is not None and stat.S_ISDIR(mode):
218
+ self._own(name)
219
+ shutil.rmtree(name, dir_fd=self.descriptor, ignore_errors=True)
220
+ return
221
+ self.unlink(name)
222
+
223
+ def _status(self, name: str) -> tuple[int | None, int]:
224
+ try:
225
+ found = os.stat(name, dir_fd=self.descriptor, follow_symlinks=False)
226
+ except OSError:
227
+ return None, 0
228
+ return found.st_mode, found.st_size