@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -37,7 +37,7 @@ the **pin** (:class:`SnapshotPin`): the hosted snapshot's ``batchDigest`` (and s
|
|
|
37
37
|
fields), which replaces the data files as the default batch. The tree's ``data/*.json``
|
|
38
38
|
then no longer describes the world's data — only that snapshot does — so a pinned tree
|
|
39
39
|
is never rebuilt from, nor served from, its data files; the platform hands the file
|
|
40
|
-
down (``
|
|
40
|
+
down (``gateway worlds sandbox open``, the pod's ``WORLD_SNAPSHOT_*``). Where the file is not
|
|
41
41
|
handed down (a sandbox), the pin itself proves which domain the hosted rows were stored
|
|
42
42
|
under, so the contract still compiles and checks.
|
|
43
43
|
|
|
@@ -49,15 +49,18 @@ from __future__ import annotations
|
|
|
49
49
|
import gzip
|
|
50
50
|
import hashlib
|
|
51
51
|
import json
|
|
52
|
+
import os
|
|
53
|
+
import re
|
|
52
54
|
import shutil
|
|
53
55
|
import urllib.request
|
|
56
|
+
import uuid
|
|
54
57
|
from dataclasses import dataclass
|
|
55
58
|
from datetime import datetime, timezone
|
|
56
59
|
from pathlib import Path
|
|
57
|
-
from typing import Any, Iterable
|
|
60
|
+
from typing import Any, Callable, Iterable
|
|
58
61
|
|
|
59
62
|
from .._toml import tomllib
|
|
60
|
-
from .compiler import LOCK, Contract,
|
|
63
|
+
from .compiler import LOCK, Contract, check_bundle
|
|
61
64
|
from .database import WorldDatabase
|
|
62
65
|
from .domain_migration import DomainMigration
|
|
63
66
|
from .validation import ContractError, canonical, fail, read_json
|
|
@@ -72,10 +75,15 @@ SIDECAR_KEYS = ("contractHash", "batchDigest", "entityCounts", "builtAt", "runti
|
|
|
72
75
|
ENV_URL = "WORLD_SNAPSHOT_URL"
|
|
73
76
|
ENV_SHA256 = "WORLD_SNAPSHOT_SHA256"
|
|
74
77
|
ENV_DIGEST = "WORLD_SNAPSHOT_DIGEST"
|
|
78
|
+
#: Where a sandbox keeps the hosted snapshots its trees pin, outside every tree (``<root>/<batchDigest>/``).
|
|
79
|
+
STORE_ENV = "GATEWAY_SNAPSHOT_STORE"
|
|
80
|
+
_BATCH_DIGEST = re.compile(r"[0-9a-f]{64}")
|
|
75
81
|
PIN_POINTER = f"/{DEFAULT_DIR}/{PIN}"
|
|
76
82
|
#: The codes a pin is refused with: never a data refusal, so never discarded as one.
|
|
77
83
|
PIN_REFUSALS = ("snapshot_pin", "snapshot_pinned")
|
|
78
84
|
REIMPORT = f"remove {DEFAULT_DIR}/{PIN} and import every row again (`gateway worlds data import <slug> <rows>`)"
|
|
85
|
+
REBUILD_REMEDY = "recompile to rebuild it from the data files, or restore the domain it was built under"
|
|
86
|
+
PINNED_REMEDY = "recompile to carry it to this contract, or restore the domain it was built under"
|
|
79
87
|
_BLOCK = 1024 * 1024
|
|
80
88
|
|
|
81
89
|
|
|
@@ -131,15 +139,15 @@ def pin_path(world: str | Path) -> Path:
|
|
|
131
139
|
|
|
132
140
|
|
|
133
141
|
def refuse_unheld_pin(world: str | Path, contract: Contract) -> None:
|
|
134
|
-
"""A pinned tree that does not hold its hosted snapshot
|
|
135
|
-
is built, copied or served from its data files
|
|
142
|
+
"""A pinned tree that does not hold its hosted snapshot, and finds no copy of it in the
|
|
143
|
+
snapshot store, has no rows here: nothing is built, copied or served from its data files
|
|
144
|
+
in their place."""
|
|
136
145
|
pin = SnapshotPin.of(world)
|
|
137
146
|
if pin is not None:
|
|
138
147
|
fail(
|
|
139
148
|
"snapshot_pinned",
|
|
140
|
-
f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold
|
|
141
|
-
|
|
142
|
-
f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files",
|
|
149
|
+
f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold"
|
|
150
|
+
+ SnapshotStore.configured().remedy(pin),
|
|
143
151
|
f"/{DEFAULT_DIR}/{snapshot_name(contract)}",
|
|
144
152
|
)
|
|
145
153
|
|
|
@@ -185,6 +193,14 @@ class SnapshotPin:
|
|
|
185
193
|
def batch_digest(self) -> str:
|
|
186
194
|
return self.fields["batchDigest"]
|
|
187
195
|
|
|
196
|
+
def held(self, contract: Contract) -> bool:
|
|
197
|
+
"""Whether the tree holds the pinned snapshot (the platform laid it beside the tree): its default
|
|
198
|
+
snapshot names this batch in the bytes its sidecar names. A pinned tree that does not hold it has
|
|
199
|
+
no rows of its own, so any snapshot in it was built from the data files the pin replaced."""
|
|
200
|
+
snapshot = default_snapshot(self.world, contract)
|
|
201
|
+
sidecar = _holding(snapshot, self.batch_digest)
|
|
202
|
+
return sidecar is not None and sidecar["fileSha256"] == file_sha256(snapshot)
|
|
203
|
+
|
|
188
204
|
def counts(self, contract: Contract) -> dict[str, int]:
|
|
189
205
|
"""The hosted snapshot's entity counts as the platform recorded them, per entity."""
|
|
190
206
|
counts = self.fields.get("entityCounts")
|
|
@@ -278,6 +294,86 @@ class SnapshotPin:
|
|
|
278
294
|
return None
|
|
279
295
|
|
|
280
296
|
|
|
297
|
+
@dataclass(frozen=True)
|
|
298
|
+
class SnapshotStore:
|
|
299
|
+
"""The hosted snapshots a sandbox keeps beside its trees, never in one (``$GATEWAY_SNAPSHOT_STORE``,
|
|
300
|
+
``<root>/<batchDigest>/<snapshot name>`` with its sidecar), where the platform lays the one a
|
|
301
|
+
sandbox's version serves. A pinned tree that holds no snapshot of its own reads its rows from the
|
|
302
|
+
copy of its batch; ``root`` is None where no store is configured or its directory does not exist
|
|
303
|
+
(a sandbox the platform lays no snapshot into)."""
|
|
304
|
+
|
|
305
|
+
root: Path | None
|
|
306
|
+
|
|
307
|
+
@classmethod
|
|
308
|
+
def configured(cls) -> "SnapshotStore":
|
|
309
|
+
root = os.environ.get(STORE_ENV)
|
|
310
|
+
return cls(Path(root) if root and Path(root).is_dir() else None)
|
|
311
|
+
|
|
312
|
+
def held(self, pin: SnapshotPin, contract: Contract) -> tuple[Path, dict] | None:
|
|
313
|
+
"""The copy of the pinned batch and its sidecar, opened under ``contract``: the copy itself when
|
|
314
|
+
its rows were stored under this contract's storage, else the copy carried to it (kept beside the
|
|
315
|
+
original) when the contract only adds to their domain. None without a copy that is the bytes its
|
|
316
|
+
sidecar names and, when the pin names them, the pin's own bytes."""
|
|
317
|
+
if self.root is None or not _BATCH_DIGEST.fullmatch(pin.batch_digest):
|
|
318
|
+
return None
|
|
319
|
+
copy = self.root / pin.batch_digest / snapshot_name(contract)
|
|
320
|
+
sidecar = _readable(lambda: _holding(copy, pin.batch_digest))
|
|
321
|
+
if sidecar is None or sidecar["fileSha256"] != file_sha256(copy):
|
|
322
|
+
return None
|
|
323
|
+
if sidecar["fileSha256"] != pin.fields.get("fileSha256", sidecar["fileSha256"]):
|
|
324
|
+
return None
|
|
325
|
+
if storage_refusal(copy, contract) is None:
|
|
326
|
+
return copy, sidecar
|
|
327
|
+
return self._carried(copy, pin, contract)
|
|
328
|
+
|
|
329
|
+
def remedy(self, pin: SnapshotPin) -> str:
|
|
330
|
+
"""How a refusal of a pinned snapshot neither the tree nor this store holds ends."""
|
|
331
|
+
if self.root is None:
|
|
332
|
+
return (
|
|
333
|
+
": run the world on the platform (`gateway worlds session open <slug>`, `gateway worlds sandbox open <slug>`), "
|
|
334
|
+
f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files"
|
|
335
|
+
)
|
|
336
|
+
return (
|
|
337
|
+
f", nor does this sandbox hold a whole copy at {self.root / pin.batch_digest}: the platform lays there, "
|
|
338
|
+
"before each `gateway worlds sandbox exec`, the snapshot this sandbox's version serves (that exec's "
|
|
339
|
+
f"warnings say why it could not). Restore {DEFAULT_DIR}/{PIN} as the version has it if it was edited, or "
|
|
340
|
+
"remove that directory and exec again; meanwhile `gateway worlds data sql <slug>` reads the rows from your "
|
|
341
|
+
"machine"
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
def _carried(self, copy: Path, pin: SnapshotPin, contract: Contract) -> tuple[Path, dict]:
|
|
345
|
+
carried = copy.parent / contract_hash(contract) / copy.name
|
|
346
|
+
kept = _readable(lambda: holds(carried, contract, pin.batch_digest))
|
|
347
|
+
if kept is not None:
|
|
348
|
+
return carried, kept
|
|
349
|
+
for other in copy.parent.iterdir():
|
|
350
|
+
if other.is_dir() and other.name != carried.parent.name:
|
|
351
|
+
shutil.rmtree(other, ignore_errors=True)
|
|
352
|
+
fresh = carried.with_name(f".{carried.name}.{uuid.uuid4().hex}")
|
|
353
|
+
fresh.parent.mkdir(parents=True, exist_ok=True)
|
|
354
|
+
WorldDatabase.clone_file(copy, fresh)
|
|
355
|
+
try:
|
|
356
|
+
DomainMigration.carry(fresh, contract, pin.world)
|
|
357
|
+
with WorldDatabase.open(fresh, contract, audit=False) as store:
|
|
358
|
+
counts = store.counts()
|
|
359
|
+
except BaseException:
|
|
360
|
+
fresh.unlink(missing_ok=True)
|
|
361
|
+
raise
|
|
362
|
+
_replace(carried, fresh)
|
|
363
|
+
return carried, write_sidecar(carried, contract_hash(contract), pin.batch_digest, counts)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _readable(read: Callable[[], dict | None]) -> dict | None:
|
|
367
|
+
"""``read()``, or None when the sidecar it reads cannot be read: a store copy is the platform's to lay
|
|
368
|
+
again and a carried copy the store's to carry again, so an unreadable sidecar there is a missing copy."""
|
|
369
|
+
try:
|
|
370
|
+
return read()
|
|
371
|
+
except ContractError as error:
|
|
372
|
+
if error.detail.get("code") != "snapshot":
|
|
373
|
+
raise
|
|
374
|
+
return None
|
|
375
|
+
|
|
376
|
+
|
|
281
377
|
def expected_default_digest(world: str | Path) -> str:
|
|
282
378
|
"""The batch digest the default snapshot must carry: the pin's for a data-only
|
|
283
379
|
version, the data files' for an authored tree."""
|
|
@@ -409,9 +505,9 @@ def storage_refusal(snapshot: Path, contract: Contract) -> ContractError | None:
|
|
|
409
505
|
return error
|
|
410
506
|
|
|
411
507
|
|
|
412
|
-
def verify(snapshot: Path, contract: Contract) -> dict:
|
|
508
|
+
def verify(snapshot: Path, contract: Contract, remedy: str = REBUILD_REMEDY) -> dict:
|
|
413
509
|
"""The sidecar of a snapshot whose bytes it names and whose rows ``contract`` may
|
|
414
|
-
open: anything else is refused."""
|
|
510
|
+
open: anything else is refused, saying what a recompile does about it."""
|
|
415
511
|
shown = _shown(snapshot)
|
|
416
512
|
sidecar = read_sidecar(snapshot)
|
|
417
513
|
if sidecar is None:
|
|
@@ -424,8 +520,7 @@ def verify(snapshot: Path, contract: Contract) -> dict:
|
|
|
424
520
|
if refusal is not None:
|
|
425
521
|
fail(
|
|
426
522
|
"snapshot",
|
|
427
|
-
f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); "
|
|
428
|
-
"recompile to rebuild it from the data files, or restore the domain it was built under",
|
|
523
|
+
f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); {remedy}",
|
|
429
524
|
shown,
|
|
430
525
|
changed=refusal.detail.get("changed", []),
|
|
431
526
|
)
|
|
@@ -523,9 +618,14 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
|
|
|
523
618
|
def build_all(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
|
|
524
619
|
"""The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
|
|
525
620
|
A pinned tree that does not hold its hosted snapshot builds none (its data lives on the
|
|
526
|
-
platform)
|
|
621
|
+
platform) and keeps none: the snapshots its data files built claim data that no longer
|
|
622
|
+
applies. The default is the pin's view, once the pin proves the rows' domain."""
|
|
527
623
|
world = Path(world)
|
|
528
624
|
pin = SnapshotPin.of(world)
|
|
625
|
+
if pin is not None and not pin.held(contract):
|
|
626
|
+
view = pin.view(contract)
|
|
627
|
+
discard_all(world)
|
|
628
|
+
return {"default": view, "scenarios": {}}
|
|
529
629
|
if pin is not None:
|
|
530
630
|
carry_held(world, contract)
|
|
531
631
|
if pin is not None and holds(default_snapshot(world, contract), contract, pin.batch_digest) is None:
|
|
@@ -572,23 +672,24 @@ def discard_all(world: str | Path) -> None:
|
|
|
572
672
|
|
|
573
673
|
|
|
574
674
|
def check_all(world: str | Path, contract: Contract) -> dict:
|
|
575
|
-
"""Every snapshot present in the tree, verified against its sidecar; absent ones are not required.
|
|
675
|
+
"""Every snapshot present in the tree, verified against its sidecar; absent ones are not required.
|
|
676
|
+
A pinned tree that does not hold its hosted snapshot has none to verify: nothing opens the
|
|
677
|
+
snapshots its data files built (compile removes them), so they are never refused."""
|
|
576
678
|
world = Path(world)
|
|
577
679
|
found: dict[str, Any] = {"default": None, "scenarios": {}}
|
|
680
|
+
pin = SnapshotPin.of(world)
|
|
681
|
+
if pin is not None and not pin.held(contract):
|
|
682
|
+
return found
|
|
683
|
+
remedy = PINNED_REMEDY if pin is not None else REBUILD_REMEDY
|
|
578
684
|
snapshot = default_snapshot(world, contract)
|
|
579
685
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
580
|
-
found["default"] = verify(snapshot, contract)
|
|
581
|
-
#
|
|
582
|
-
|
|
583
|
-
# replays the journals past a stale snapshot (WorldState opens only a
|
|
584
|
-
# current one). `stale` is in every check report so it never passes unseen.
|
|
585
|
-
found["default"]["stale"] = SnapshotPin.of(world) is None and found["default"][
|
|
586
|
-
"batchDigest"
|
|
587
|
-
] != default_batch_digest(world)
|
|
686
|
+
found["default"] = verify(snapshot, contract, remedy)
|
|
687
|
+
# Reported, not refused: a served world replays the journals past a stale snapshot.
|
|
688
|
+
found["default"]["stale"] = pin is None and found["default"]["batchDigest"] != default_batch_digest(world)
|
|
588
689
|
for name in scenario_dirs(world):
|
|
589
690
|
snapshot = scenario_snapshot(world, name, contract)
|
|
590
691
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
591
|
-
found["scenarios"][name] = verify(snapshot, contract)
|
|
692
|
+
found["scenarios"][name] = verify(snapshot, contract, remedy)
|
|
592
693
|
return found
|
|
593
694
|
|
|
594
695
|
|
|
@@ -623,24 +724,40 @@ def current_default(world: str | Path, contract: Contract) -> dict | None:
|
|
|
623
724
|
return holds(snapshot, contract, expected_default_digest(world))
|
|
624
725
|
|
|
625
726
|
|
|
727
|
+
def held_default(world: str | Path, contract: Contract) -> tuple[Path, dict] | None:
|
|
728
|
+
"""The default snapshot a reader opens, and its sidecar: the tree's own when it holds the tree's
|
|
729
|
+
data (or the pin's), else, for a pinned tree, the snapshot store's copy of the pinned batch."""
|
|
730
|
+
world = Path(world)
|
|
731
|
+
sidecar = current_default(world, contract)
|
|
732
|
+
if sidecar is not None:
|
|
733
|
+
return default_snapshot(world, contract), sidecar
|
|
734
|
+
pin = SnapshotPin.of(world)
|
|
735
|
+
if pin is None:
|
|
736
|
+
return None
|
|
737
|
+
return SnapshotStore.configured().held(pin, contract)
|
|
738
|
+
|
|
739
|
+
|
|
626
740
|
@dataclass(frozen=True)
|
|
627
741
|
class ServedSnapshot:
|
|
628
742
|
"""What a served world opens: its contract, the tree checked against it, and the
|
|
629
|
-
|
|
630
|
-
rows the contract may open, None on the legacy path. A pinned tree that holds
|
|
631
|
-
snapshot
|
|
632
|
-
immutable tree once and hands the answer to
|
|
743
|
+
default snapshot (``path`` and its ``sidecar``) when one holds the tree's data (or the
|
|
744
|
+
pin's) in rows the contract may open, None on the legacy path. A pinned tree that holds
|
|
745
|
+
no snapshot, and finds none in the snapshot store, is refused. Checking hashes the
|
|
746
|
+
snapshot, so a version cache checks its immutable tree once and hands the answer to
|
|
747
|
+
every session of the version."""
|
|
633
748
|
|
|
634
749
|
contract: Contract
|
|
635
750
|
sidecar: dict | None
|
|
751
|
+
path: Path | None = None
|
|
636
752
|
|
|
637
753
|
@classmethod
|
|
638
754
|
def of(cls, world: str | Path) -> "ServedSnapshot":
|
|
639
755
|
contract = check_bundle(world)
|
|
640
|
-
|
|
641
|
-
if
|
|
756
|
+
held = held_default(world, contract)
|
|
757
|
+
if held is None:
|
|
642
758
|
refuse_unheld_pin(world, contract)
|
|
643
|
-
|
|
759
|
+
return cls(contract, None)
|
|
760
|
+
return cls(contract, held[1], held[0])
|
|
644
761
|
|
|
645
762
|
|
|
646
763
|
def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest: str) -> dict:
|
|
@@ -699,6 +816,8 @@ __all__ = [
|
|
|
699
816
|
"SIDECAR",
|
|
700
817
|
"ServedSnapshot",
|
|
701
818
|
"SnapshotPin",
|
|
819
|
+
"SnapshotStore",
|
|
820
|
+
"STORE_ENV",
|
|
702
821
|
"batch_digest",
|
|
703
822
|
"build_all",
|
|
704
823
|
"build_default",
|
|
@@ -713,6 +832,7 @@ __all__ = [
|
|
|
713
832
|
"expected_default_digest",
|
|
714
833
|
"fetch",
|
|
715
834
|
"file_sha256",
|
|
835
|
+
"held_default",
|
|
716
836
|
"holds",
|
|
717
837
|
"lock_hash",
|
|
718
838
|
"migration_view",
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The files under the World Host's request ledger spill, as tenant code could have left them: version
|
|
2
|
+
workers share the host's user and cache, so every name is reached from a directory held open by
|
|
3
|
+
descriptor and never through a link, a file is written only when it is a regular one and opened without
|
|
4
|
+
waiting on a FIFO, a directory the host keeps is given back its owner's access before it is used and
|
|
5
|
+
again whenever tenant code took it, and every line carries a seal only the host can make
|
|
6
|
+
(:class:`SpillSeal`)."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import errno
|
|
11
|
+
import hashlib
|
|
12
|
+
import hmac
|
|
13
|
+
import os
|
|
14
|
+
import shutil
|
|
15
|
+
import stat
|
|
16
|
+
import uuid
|
|
17
|
+
from collections.abc import Callable, Iterator
|
|
18
|
+
from contextlib import contextmanager
|
|
19
|
+
from functools import partial
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import TypeVar
|
|
22
|
+
|
|
23
|
+
from .call_log import read_tail
|
|
24
|
+
from .handler_errors import log_line
|
|
25
|
+
|
|
26
|
+
TOMBSTONE = ".gone-"
|
|
27
|
+
DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK
|
|
28
|
+
WRITING = os.O_WRONLY | os.O_NOFOLLOW | os.O_NONBLOCK
|
|
29
|
+
Done = TypeVar("Done")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _write_all(descriptor: int, data: bytes) -> None:
|
|
33
|
+
view = memoryview(data)
|
|
34
|
+
while view:
|
|
35
|
+
view = view[os.write(descriptor, view) :]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SpillSeal:
|
|
39
|
+
"""The seal on every spill line: an HMAC under a key derived from the control-plane secret, which a
|
|
40
|
+
version worker is never given, over the account's ``source``, the line's ``place`` (a segment line's
|
|
41
|
+
segment and ordinal, a record line's predecessor's tag) and the line, so a line written, changed,
|
|
42
|
+
copied or moved by anything but this host fails it."""
|
|
43
|
+
|
|
44
|
+
OVERHEAD = 66
|
|
45
|
+
|
|
46
|
+
def __init__(self, secret: str) -> None:
|
|
47
|
+
self._key = hmac.new(secret.encode("utf-8"), b"world-host-ledger-spill/2", hashlib.sha256).digest()
|
|
48
|
+
|
|
49
|
+
def seal(self, source: str, data: bytes, place: bytes = b"") -> bytes:
|
|
50
|
+
return self._tag(source, place, data) + b" " + data + b"\n"
|
|
51
|
+
|
|
52
|
+
def opened(self, source: str, line: bytes, place: bytes = b"") -> bytes | None:
|
|
53
|
+
tag, _, data = line.partition(b" ")
|
|
54
|
+
return data if hmac.compare_digest(tag, self._tag(source, place, data)) else None
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
def tag(line: bytes) -> bytes:
|
|
58
|
+
return line.partition(b" ")[0]
|
|
59
|
+
|
|
60
|
+
def _tag(self, source: str, place: bytes, data: bytes) -> bytes:
|
|
61
|
+
message = source.encode("ascii") + b"\n" + place + b"\n" + data
|
|
62
|
+
return hmac.new(self._key, message, hashlib.sha256).hexdigest().encode("ascii")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class SpillDirectory:
|
|
66
|
+
"""A directory held open by descriptor: every name in it is reached without following a link, every
|
|
67
|
+
file read as tenant code could have written it and written only as a regular file."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, descriptor: int) -> None:
|
|
70
|
+
self.descriptor = descriptor
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def opened(cls, name: str | Path, within: int | None = None) -> SpillDirectory:
|
|
74
|
+
return cls(os.open(name, DIRECTORY, dir_fd=within))
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def owned(cls, name: str, within: int) -> SpillDirectory:
|
|
78
|
+
"""``name`` opened and given its owner's access alone; one the host cannot make its own raises."""
|
|
79
|
+
directory = cls.opened(name, within)
|
|
80
|
+
try:
|
|
81
|
+
directory.own()
|
|
82
|
+
except OSError:
|
|
83
|
+
directory.close()
|
|
84
|
+
raise
|
|
85
|
+
return directory
|
|
86
|
+
|
|
87
|
+
def own(self) -> None:
|
|
88
|
+
"""Its owner's access alone again, through the descriptor it is held by: tenant code can change
|
|
89
|
+
a held directory's mode, never which directory is held."""
|
|
90
|
+
os.fchmod(self.descriptor, 0o700)
|
|
91
|
+
|
|
92
|
+
def reowned(self, operation: Callable[[], Done]) -> Done:
|
|
93
|
+
"""``operation`` on a name in this directory, run once more after the directory is made the host's
|
|
94
|
+
own again (:meth:`own`) when tenant code took the host's access to it, its write or search bit."""
|
|
95
|
+
try:
|
|
96
|
+
return operation()
|
|
97
|
+
except PermissionError:
|
|
98
|
+
self.own()
|
|
99
|
+
return operation()
|
|
100
|
+
|
|
101
|
+
def close(self) -> None:
|
|
102
|
+
os.close(self.descriptor)
|
|
103
|
+
|
|
104
|
+
@contextmanager
|
|
105
|
+
def child(self, name: str) -> Iterator[SpillDirectory]:
|
|
106
|
+
directory = self.reowned(partial(SpillDirectory.owned, name, self.descriptor))
|
|
107
|
+
try:
|
|
108
|
+
yield directory
|
|
109
|
+
finally:
|
|
110
|
+
directory.close()
|
|
111
|
+
|
|
112
|
+
def names(self) -> list[str]:
|
|
113
|
+
return sorted(os.listdir(self.descriptor))
|
|
114
|
+
|
|
115
|
+
def mode(self, name: str) -> int | None:
|
|
116
|
+
mode, _ = self._status(name)
|
|
117
|
+
return mode
|
|
118
|
+
|
|
119
|
+
def size(self, name: str) -> int | None:
|
|
120
|
+
mode, size = self._status(name)
|
|
121
|
+
return size if mode is not None and stat.S_ISREG(mode) else None
|
|
122
|
+
|
|
123
|
+
def ensure(self, name: str) -> None:
|
|
124
|
+
"""A directory ``name`` here, made when missing (:meth:`child` refuses anything else at it)."""
|
|
125
|
+
try:
|
|
126
|
+
os.mkdir(name, 0o700, dir_fd=self.descriptor)
|
|
127
|
+
except FileExistsError:
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
def make(self, name: str) -> None:
|
|
131
|
+
os.mkdir(name, 0o700, dir_fd=self.descriptor)
|
|
132
|
+
|
|
133
|
+
def create(self, name: str, data: bytes) -> None:
|
|
134
|
+
"""``data`` as a new regular file ``name``; refused when anything, a link included, is there."""
|
|
135
|
+
descriptor = self._writable(name, os.O_CREAT | os.O_EXCL)
|
|
136
|
+
try:
|
|
137
|
+
_write_all(descriptor, data)
|
|
138
|
+
except OSError:
|
|
139
|
+
self.unlink(name)
|
|
140
|
+
raise
|
|
141
|
+
finally:
|
|
142
|
+
os.close(descriptor)
|
|
143
|
+
|
|
144
|
+
def add(self, name: str, data: bytes) -> None:
|
|
145
|
+
"""``data`` appended to the regular file ``name``, made when missing."""
|
|
146
|
+
descriptor = self._writable(name, os.O_CREAT | os.O_APPEND)
|
|
147
|
+
try:
|
|
148
|
+
_write_all(descriptor, data)
|
|
149
|
+
finally:
|
|
150
|
+
os.close(descriptor)
|
|
151
|
+
|
|
152
|
+
def replace(self, name: str, data: bytes) -> None:
|
|
153
|
+
"""``data`` as the whole of ``name`` at once: written beside it, then renamed over it."""
|
|
154
|
+
written = f".{name}-{uuid.uuid4().hex}"
|
|
155
|
+
self.create(written, data)
|
|
156
|
+
try:
|
|
157
|
+
os.rename(written, name, src_dir_fd=self.descriptor, dst_dir_fd=self.descriptor)
|
|
158
|
+
except OSError:
|
|
159
|
+
self.unlink(written)
|
|
160
|
+
raise
|
|
161
|
+
|
|
162
|
+
def tail(self, name: str, max_bytes: int) -> bytes | None:
|
|
163
|
+
"""The newest ``max_bytes`` of ``name`` through :func:`~.call_log.read_tail` (anything but a regular
|
|
164
|
+
file reads as empty); None when it cannot be opened or read."""
|
|
165
|
+
try:
|
|
166
|
+
return read_tail(name, max_bytes, self.descriptor)
|
|
167
|
+
except OSError:
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
def unlink(self, name: str) -> None:
|
|
171
|
+
try:
|
|
172
|
+
os.unlink(name, dir_fd=self.descriptor)
|
|
173
|
+
except FileNotFoundError:
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
def discard(self, name: str) -> None:
|
|
177
|
+
"""Move whatever is at ``name`` out of the way at once (a link as a link, a directory given its owner's
|
|
178
|
+
access first, which moving one needs on some kernels); :meth:`sweep` deletes it."""
|
|
179
|
+
if stat.S_ISDIR(self.mode(name) or 0):
|
|
180
|
+
self._own(name)
|
|
181
|
+
try:
|
|
182
|
+
os.rename(name, f"{TOMBSTONE}{uuid.uuid4().hex}", src_dir_fd=self.descriptor, dst_dir_fd=self.descriptor)
|
|
183
|
+
except FileNotFoundError:
|
|
184
|
+
return None
|
|
185
|
+
|
|
186
|
+
def sweep(self) -> None:
|
|
187
|
+
"""Delete what :meth:`discard` set aside, as far as it can; what it cannot is logged and left."""
|
|
188
|
+
for name in (name for name in self.names() if name.startswith(TOMBSTONE)):
|
|
189
|
+
self._delete(name)
|
|
190
|
+
|
|
191
|
+
def _writable(self, name: str, flags: int) -> int:
|
|
192
|
+
descriptor = os.open(name, WRITING | flags, 0o600, dir_fd=self.descriptor)
|
|
193
|
+
try:
|
|
194
|
+
regular = stat.S_ISREG(os.fstat(descriptor).st_mode)
|
|
195
|
+
except OSError:
|
|
196
|
+
os.close(descriptor)
|
|
197
|
+
raise
|
|
198
|
+
if regular:
|
|
199
|
+
return descriptor
|
|
200
|
+
os.close(descriptor)
|
|
201
|
+
raise OSError(errno.EINVAL, "not a regular file", name)
|
|
202
|
+
|
|
203
|
+
def _own(self, name: str) -> None:
|
|
204
|
+
try:
|
|
205
|
+
os.chmod(name, 0o700, dir_fd=self.descriptor)
|
|
206
|
+
except OSError:
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
def _delete(self, name: str) -> None:
|
|
210
|
+
mode, _ = self._status(name)
|
|
211
|
+
try:
|
|
212
|
+
self._remove(name, mode)
|
|
213
|
+
except OSError as error:
|
|
214
|
+
log_line(f"request ledger spill: {name} not removed: {type(error).__name__}")
|
|
215
|
+
|
|
216
|
+
def _remove(self, name: str, mode: int | None) -> None:
|
|
217
|
+
if mode is not None and stat.S_ISDIR(mode):
|
|
218
|
+
self._own(name)
|
|
219
|
+
shutil.rmtree(name, dir_fd=self.descriptor, ignore_errors=True)
|
|
220
|
+
return
|
|
221
|
+
self.unlink(name)
|
|
222
|
+
|
|
223
|
+
def _status(self, name: str) -> tuple[int | None, int]:
|
|
224
|
+
try:
|
|
225
|
+
found = os.stat(name, dir_fd=self.descriptor, follow_symlinks=False)
|
|
226
|
+
except OSError:
|
|
227
|
+
return None, 0
|
|
228
|
+
return found.st_mode, found.st_size
|