@withgateway/sdk 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1798 -1713
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2412 -2327
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +47 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +43 -27
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +390 -287
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +375 -271
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +36 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +32 -16
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +2 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +2 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +26 -22
- package/dist/worlds/index.d.ts +26 -22
- package/dist/worlds/index.js +307 -203
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +297 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +2 -1
- package/runtime/gatewaysdk/environment/_bundle.py +14 -24
- package/runtime/gatewaysdk/environment/schema/api.py +37 -18
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +156 -546
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +6 -4
- package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
- package/runtime/gatewaysdk/environment/world.py +391 -0
- package/skills/hooks-getting-started/SKILL.md +6 -4
|
@@ -34,8 +34,6 @@ import csv
|
|
|
34
34
|
import hashlib
|
|
35
35
|
import io
|
|
36
36
|
import json
|
|
37
|
-
import shutil
|
|
38
|
-
import tempfile
|
|
39
37
|
from pathlib import Path
|
|
40
38
|
from typing import Any, Callable, Iterable
|
|
41
39
|
|
|
@@ -377,7 +375,7 @@ def ingest_records(
|
|
|
377
375
|
``batch`` from the import; ``dry_run`` and ``redact`` say how it ran. With
|
|
378
376
|
any refusal nothing is imported.
|
|
379
377
|
"""
|
|
380
|
-
from .data import
|
|
378
|
+
from .data import import_rows
|
|
381
379
|
from .mapping import FILE, Mapping, apply_mapping, load_mapping, parse_mapping, validate_mapping
|
|
382
380
|
from .redaction import parse_mode
|
|
383
381
|
from .transform import DEFAULT_TIMEOUT, check_rows, transform_records
|
|
@@ -448,20 +446,14 @@ def ingest_records(
|
|
|
448
446
|
return report
|
|
449
447
|
# The contract's own refusal of a row raises from the import, as `data check`
|
|
450
448
|
# raises it: the CLI turns the bridge's rejection into one refusal naming the pointer.
|
|
451
|
-
|
|
452
|
-
with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
|
|
453
|
-
staged = Path(scratch) / "world"
|
|
454
|
-
shutil.copytree(root, staged, ignore=shutil.ignore_patterns(*SCRATCH_IGNORE))
|
|
455
|
-
proven = import_rows(staged, projected, mode=mode, redact=redact)
|
|
456
|
-
report.update({"kind": proven["kind"], "entity_counts": proven["entity_counts"], "batch": proven.get("batch")})
|
|
457
|
-
return report
|
|
458
|
-
result = import_rows(root, projected, mode=mode, redact=redact)
|
|
449
|
+
result = import_rows(root, projected, mode=mode, redact=redact, dry_run=dry_run)
|
|
459
450
|
report.update(
|
|
460
451
|
{
|
|
461
452
|
"kind": result["kind"],
|
|
462
453
|
"entity_counts": result["entity_counts"],
|
|
463
454
|
"batch": result.get("batch"),
|
|
464
455
|
"changed": result["changed"],
|
|
456
|
+
**{key: result[key] for key in ("proof", "deferred") if key in result},
|
|
465
457
|
}
|
|
466
458
|
)
|
|
467
459
|
return report
|
|
@@ -37,12 +37,13 @@ from pathlib import Path
|
|
|
37
37
|
from typing import Any, Callable
|
|
38
38
|
|
|
39
39
|
from . import snapshot as snapshots
|
|
40
|
-
from .api import Router, WorldApp, load_api_settings,
|
|
41
|
-
from .host_surfaces import UiSurface, api_target, load_ui_surface, static_file
|
|
40
|
+
from .api import Router, WorldApp, load_api_settings, send_response, vendor_credential, with_query
|
|
42
41
|
from .compiler import bundle_mode, fail
|
|
43
42
|
from .conform import PIN
|
|
44
43
|
from .connector import SPEC, load_spec
|
|
45
44
|
from .dependencies import local_dependency_trees
|
|
45
|
+
from .host_surfaces import UiSurface, api_target, load_ui_surface, static_file
|
|
46
|
+
from .runtime import call_operation, client_grant, task_card
|
|
46
47
|
from .tools_world import (
|
|
47
48
|
DependencyWorld,
|
|
48
49
|
ToolsTask,
|
|
@@ -52,30 +53,11 @@ from .tools_world import (
|
|
|
52
53
|
load_tools_version,
|
|
53
54
|
)
|
|
54
55
|
from .validation import ContractError
|
|
56
|
+
from .verbs import refusal, session_verb, unknown_verb
|
|
55
57
|
|
|
56
58
|
LOCAL_TASK = ToolsTask(id="local", prompt="A local session; every verifier grades.")
|
|
57
59
|
|
|
58
60
|
|
|
59
|
-
def refusal(error: ContractError) -> dict:
|
|
60
|
-
"""A refused call as the answer a hosted session gives it: an unknown tool,
|
|
61
|
-
arguments off the schema, a tool outside the task's allow-list, a write the
|
|
62
|
-
store will not take, all ``{status: 400, body: {error, detail}}``."""
|
|
63
|
-
return {"status": 400, "body": {"error": error.detail.get("code", "host"), "detail": str(error)}}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def seed_args(body: dict) -> dict:
|
|
67
|
-
"""The seed a ``POST /__session/seed`` body asks for: ``{rows, mode}``, the
|
|
68
|
-
same under ``args``, or bare ``{entity: [row, ...]}`` — the shape a rows
|
|
69
|
-
file has — taken as the rows of an append."""
|
|
70
|
-
if isinstance(body.get("args"), dict):
|
|
71
|
-
body = body["args"]
|
|
72
|
-
if "rows" in body or "mode" in body:
|
|
73
|
-
return body
|
|
74
|
-
if body and all(isinstance(value, list) for value in body.values()):
|
|
75
|
-
return {"rows": body}
|
|
76
|
-
return body
|
|
77
|
-
|
|
78
|
-
|
|
79
61
|
def has_http(root: str | Path) -> bool:
|
|
80
62
|
"""Whether the world serves HTTP at all: only a tree with ``connector.toml``
|
|
81
63
|
has routes. A schema world without one is tools-only, and that is a whole
|
|
@@ -344,40 +326,6 @@ SESSION_PREFIX = "/__session/"
|
|
|
344
326
|
UI_PREFIX = "/__ui/"
|
|
345
327
|
|
|
346
328
|
|
|
347
|
-
def tool_contracts(bundle: Path, prefix: str = "") -> dict[str, dict]:
|
|
348
|
-
"""Each compiled tool's ``{description, input}`` from the bundle's
|
|
349
|
-
``tool-schemas.json`` (refs inlined by the compiler), keyed by served name."""
|
|
350
|
-
path = bundle / "tool-schemas.json"
|
|
351
|
-
if not path.is_file():
|
|
352
|
-
return {}
|
|
353
|
-
tools = json.loads(path.read_text(encoding="utf-8")).get("tools") or {}
|
|
354
|
-
return {
|
|
355
|
-
f"{prefix}{name}": {"description": str(entry.get("description") or ""), "input": entry.get("input_schema")}
|
|
356
|
-
for name, entry in tools.items()
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
def task_card(world: ToolsWorld) -> dict:
|
|
361
|
-
"""What an outside agent is handed about the session's task: ``{task,
|
|
362
|
-
prompt, tools, contracts, verifiers, worlds}`` — ``tools`` the served names,
|
|
363
|
-
``contracts`` each one's description and input schema, ``verifiers`` the
|
|
364
|
-
root's that grade, ``worlds`` each dependency alias to the verifiers of its
|
|
365
|
-
own that grade (``slack/message_posted`` in the grade's ``rewards``)."""
|
|
366
|
-
task = world.task
|
|
367
|
-
contracts = tool_contracts(world.version.bundle)
|
|
368
|
-
for alias, dep in world.dependencies.items():
|
|
369
|
-
contracts.update(tool_contracts(dep.version.bundle, f"{alias}."))
|
|
370
|
-
return {
|
|
371
|
-
"task": task.id,
|
|
372
|
-
"prompt": task.prompt,
|
|
373
|
-
"tools": list(world.served),
|
|
374
|
-
"contracts": {name: contracts[name] for name in world.served if name in contracts},
|
|
375
|
-
"verifiers": task.selected(world.version.verifiers),
|
|
376
|
-
"worlds": {alias: task.selected_of(alias, dep.version.verifiers) for alias, dep in world.dependencies.items()},
|
|
377
|
-
"metadata": dict(task.metadata),
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
|
|
381
329
|
def make_session_handler(
|
|
382
330
|
world: ToolsWorld,
|
|
383
331
|
app: WorldApp | None,
|
|
@@ -403,21 +351,33 @@ def make_session_handler(
|
|
|
403
351
|
# The page's calls and a `/__session/call` of an operation are presented
|
|
404
352
|
# as a granted client, as the host presents them; the world's own token
|
|
405
353
|
# dance stays available beside.
|
|
406
|
-
grant =
|
|
354
|
+
grant = client_grant(app, api_key)
|
|
407
355
|
|
|
408
|
-
def close(
|
|
356
|
+
def close() -> dict:
|
|
409
357
|
closing.set()
|
|
410
358
|
return {"closed": True, "task": world.task.id}
|
|
411
359
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
("
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
(
|
|
360
|
+
def seed(rows: dict, mode: str, redact: str) -> dict:
|
|
361
|
+
if redact != "off":
|
|
362
|
+
fail("host", "a schema world has no [redact] policy; seed it with redact=off")
|
|
363
|
+
return world.seed({"rows": rows, "mode": mode})
|
|
364
|
+
|
|
365
|
+
def grade(report: str | None = None) -> dict:
|
|
366
|
+
# The local session grades by the world's own verifiers; an agent's
|
|
367
|
+
# report is read by a session task's grader, which only the host runs.
|
|
368
|
+
return world.grade()
|
|
369
|
+
|
|
370
|
+
# The session protocol (:data:`verbs.SESSION_VERBS`) bound to this
|
|
371
|
+
# session's operations, by route; the host binds the same rows to its own.
|
|
372
|
+
operations: dict[str, Callable[..., dict]] = {
|
|
373
|
+
"GET tools": lambda: {"tools": list(world.served)},
|
|
374
|
+
"GET task": lambda: task_card(world),
|
|
375
|
+
"GET state": world.state,
|
|
376
|
+
"POST grade": grade,
|
|
377
|
+
"POST reset": world.reset,
|
|
378
|
+
"POST seed": seed,
|
|
379
|
+
"POST call": lambda tool, args: call_served(world, app, tool, args, api_key=api_key, grant=grant),
|
|
380
|
+
"POST close": close,
|
|
421
381
|
}
|
|
422
382
|
|
|
423
383
|
class Handler(BaseHTTPRequestHandler):
|
|
@@ -498,14 +458,11 @@ def make_session_handler(
|
|
|
498
458
|
return self._reply(400, {"error": "malformed json", "detail": str(error)})
|
|
499
459
|
if not isinstance(body, dict):
|
|
500
460
|
return self._reply(400, {"error": "malformed json", "detail": "the body must be a json object"})
|
|
501
|
-
|
|
502
|
-
if
|
|
503
|
-
|
|
504
|
-
return self._reply(
|
|
505
|
-
404, {"error": f"unknown session route {method} {SESSION_PREFIX}{verb}; one of {known}"}
|
|
506
|
-
)
|
|
461
|
+
row = session_verb(method, verb or None)
|
|
462
|
+
if row is None or row.hosted:
|
|
463
|
+
return self._reply(404, unknown_verb(method, verb or None, hosted=False, prefix=SESSION_PREFIX))
|
|
507
464
|
try:
|
|
508
|
-
return self._reply(200,
|
|
465
|
+
return self._reply(200, operations[row.route](**row.read(body)))
|
|
509
466
|
except ContractError as error:
|
|
510
467
|
return self._reply(**refusal(error))
|
|
511
468
|
|
|
@@ -541,16 +498,6 @@ def _local_origin(origin: str) -> bool:
|
|
|
541
498
|
return host in ("127.0.0.1", "localhost", "::1")
|
|
542
499
|
|
|
543
500
|
|
|
544
|
-
def local_grant(app: WorldApp | None, api_key: str | None) -> str | None:
|
|
545
|
-
"""The bearer an ``oauth2`` world's routes take when the session itself
|
|
546
|
-
calls them (the page's api prefix, an operation by name); None for any
|
|
547
|
-
other scheme, or when the world refuses the client-credentials grant."""
|
|
548
|
-
if app is None or app.router.spec.scheme != "oauth2":
|
|
549
|
-
return None
|
|
550
|
-
issued = app.router.issue_token({"grant_type": "client_credentials", "client_secret": api_key or ""}, api_key)
|
|
551
|
-
return issued["access_token"] if issued else None
|
|
552
|
-
|
|
553
|
-
|
|
554
501
|
def call_served(
|
|
555
502
|
world: ToolsWorld,
|
|
556
503
|
app: WorldApp | None,
|
|
@@ -566,51 +513,10 @@ def call_served(
|
|
|
566
513
|
same store, ``{status, body}`` the route's own. A name that is both is the
|
|
567
514
|
tool's."""
|
|
568
515
|
if app is not None and tool not in world.version.tool_names and tool in app.router.spec.operations:
|
|
569
|
-
return call_operation(
|
|
516
|
+
return call_operation(app, tool, args, api_key or "", allowed=world.allowed, grant=grant)
|
|
570
517
|
return world.call(tool, args)
|
|
571
518
|
|
|
572
519
|
|
|
573
|
-
def call_operation(
|
|
574
|
-
world: ToolsWorld,
|
|
575
|
-
app: WorldApp,
|
|
576
|
-
name: str,
|
|
577
|
-
args: dict,
|
|
578
|
-
*,
|
|
579
|
-
api_key: str | None = PIN,
|
|
580
|
-
grant: str | None = None,
|
|
581
|
-
) -> dict:
|
|
582
|
-
"""An operation of connector.toml called by name (:func:`operation_request`
|
|
583
|
-
reads the args); the world's own credential is presented, as the host
|
|
584
|
-
presents a session's. The answer is the route's ``{status, body}``."""
|
|
585
|
-
operation = app.router.spec.operations.get(name)
|
|
586
|
-
if operation is None:
|
|
587
|
-
known = ", ".join(sorted(app.router.spec.operations)) or "(none)"
|
|
588
|
-
fail("host", f"unknown tool '{name}'; this world serves: {known}")
|
|
589
|
-
if world.allowed is not None and name not in world.allowed:
|
|
590
|
-
fail(
|
|
591
|
-
"tool_not_allowed",
|
|
592
|
-
f"tool '{name}' is not allowed by this session's task; it allows: {', '.join(sorted(world.allowed))}",
|
|
593
|
-
)
|
|
594
|
-
method, target, raw = operation_request(app.router.spec, operation, args)
|
|
595
|
-
headers, credential_query = vendor_credential(app.router.spec, api_key or "", grant=grant)
|
|
596
|
-
status, answer, _ = app.handle(
|
|
597
|
-
method, with_query(target, credential_query), {"content-type": "application/json", **headers}, raw, "127.0.0.1"
|
|
598
|
-
)
|
|
599
|
-
return {"status": status, "body": answer}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
def _call(
|
|
603
|
-
world: ToolsWorld, app: WorldApp | None, body: dict, *, api_key: str | None = PIN, grant: str | None = None
|
|
604
|
-
) -> dict:
|
|
605
|
-
tool = body.get("tool")
|
|
606
|
-
if not isinstance(tool, str) or not tool:
|
|
607
|
-
fail("host", "tool must be a tool name (root) or alias.tool (a dependency's)")
|
|
608
|
-
args = body.get("args") or {}
|
|
609
|
-
if not isinstance(args, dict):
|
|
610
|
-
fail("host", "args must be an object of the tool's arguments")
|
|
611
|
-
return call_served(world, app, tool, args, api_key=api_key, grant=grant)
|
|
612
|
-
|
|
613
|
-
|
|
614
520
|
def _apply_scenario(world: ToolsWorld, root: Path, name: str) -> None:
|
|
615
521
|
scenarios = snapshots.scenario_dirs(root)
|
|
616
522
|
if name not in scenarios:
|
|
@@ -22,6 +22,7 @@ from pathlib import Path, PurePosixPath
|
|
|
22
22
|
from typing import TYPE_CHECKING, Any
|
|
23
23
|
|
|
24
24
|
from .._materialize import materialize_template
|
|
25
|
+
from ..world import WorldTree
|
|
25
26
|
from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle
|
|
26
27
|
from . import snapshot as snapshots
|
|
27
28
|
from .compiler import retarget_empty_journals
|
|
@@ -548,21 +549,13 @@ def _handler_execution(bundle: Path, execute: bool) -> dict[str, Any]:
|
|
|
548
549
|
return {"status": "ok", "tools": session.tools}
|
|
549
550
|
|
|
550
551
|
|
|
551
|
-
API_BUNDLE = "gateway-api-world.toml"
|
|
552
|
-
BUNDLED_WORLD = "server/world"
|
|
553
|
-
|
|
554
|
-
|
|
555
552
|
def world_root(root: Path) -> Path:
|
|
556
|
-
"""The world inside the tree the platform holds
|
|
557
|
-
world as its api bundle, so a version's
|
|
558
|
-
``seed/`` + ``scenarios/`` +
|
|
559
|
-
world
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
(nested / "connector.toml").is_file() or (nested / "schema/world.json").is_file()
|
|
563
|
-
):
|
|
564
|
-
return nested
|
|
565
|
-
return root
|
|
553
|
+
"""The world inside the tree the platform holds (``WorldTree.world``).
|
|
554
|
+
``bench push`` uploads a schema world as its api bundle, so a version's
|
|
555
|
+
tree is ``gateway-api-world.toml`` + ``seed/`` + ``scenarios/`` +
|
|
556
|
+
``server/world/<the authoring world>``: that nested world is what data
|
|
557
|
+
commands operate on. A plain tree is its own root."""
|
|
558
|
+
return WorldTree.from_dir(root).world
|
|
566
559
|
|
|
567
560
|
|
|
568
561
|
def _write_files(root: Path, entries: list) -> None:
|
|
@@ -968,12 +961,17 @@ def _import_data(request: dict) -> dict:
|
|
|
968
961
|
if merge not in MERGES:
|
|
969
962
|
fail("input", f"merge must be one of {', '.join(MERGES)}")
|
|
970
963
|
redact = parse_mode(request.get("redact"))
|
|
964
|
+
dry_run = _flag(request, "dry_run", False)
|
|
971
965
|
with tempfile.TemporaryDirectory(prefix="gateway-import-") as directory:
|
|
972
966
|
root = Path(directory) / "bundle"
|
|
973
967
|
root.mkdir()
|
|
974
968
|
_write_files(root, request.get("files"))
|
|
975
969
|
world = world_root(root)
|
|
976
|
-
result = import_rows(world, rows, mode=mode, merge=merge, redact=redact)
|
|
970
|
+
result = import_rows(world, rows, mode=mode, merge=merge, redact=redact, dry_run=dry_run)
|
|
971
|
+
keys = ("kind", "entity_counts", "changed")
|
|
972
|
+
report = {"status": "ok", **{key: result[key] for key in keys}}
|
|
973
|
+
if dry_run:
|
|
974
|
+
return {**report, **{key: result[key] for key in ("proof", "deferred") if key in result}}
|
|
977
975
|
if result["kind"] == KIND_SCHEMA and snapshots.default_snapshot(world).is_file():
|
|
978
976
|
# The journals grew; a snapshot built before this import would be
|
|
979
977
|
# served over them. Rebuild so the tree that travels back is one thing.
|
|
@@ -986,13 +984,7 @@ def _import_data(request: dict) -> dict:
|
|
|
986
984
|
from .api import write_base_seed
|
|
987
985
|
|
|
988
986
|
write_base_seed(world, root)
|
|
989
|
-
return {
|
|
990
|
-
"status": "ok",
|
|
991
|
-
"kind": result["kind"],
|
|
992
|
-
"entity_counts": result["entity_counts"],
|
|
993
|
-
"changed": result["changed"],
|
|
994
|
-
"files": _tree(root),
|
|
995
|
-
}
|
|
987
|
+
return {**report, "files": _tree(root)}
|
|
996
988
|
|
|
997
989
|
|
|
998
990
|
def _ingest_calls(request: dict) -> dict:
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Proving rows against a world without writing them: the dry run behind
|
|
2
|
+
``data check``, ``data import --dry-run``, ``ingest --dry-run`` and the bridge.
|
|
3
|
+
|
|
4
|
+
What the rows are proven *against* depends on where the world's data lives, and
|
|
5
|
+
:meth:`RowProof.of` picks the one proof that tells the truth about it:
|
|
6
|
+
|
|
7
|
+
* :class:`FilesProof` — an authored tree (no ``db/snapshot.json``): the data files
|
|
8
|
+
are the data, so the rows are imported into a scratch copy of the tree through
|
|
9
|
+
the same gate the real import runs;
|
|
10
|
+
* :class:`SnapshotProof` — a checkout pinned to a hosted snapshot that carries the
|
|
11
|
+
snapshot itself (``db/state.sqlite`` current against the pin): the platform's
|
|
12
|
+
batch lane runs over a private copy of those bytes, so references, unique keys
|
|
13
|
+
and merges are checked against the rows the platform holds;
|
|
14
|
+
* :class:`ContractProof` — a pinned checkout without its snapshot: only what needs
|
|
15
|
+
no state is proven here (shape, keys, redaction); references and unique keys are
|
|
16
|
+
the platform batch's to decide, and the report says so under ``deferred``.
|
|
17
|
+
|
|
18
|
+
Dropping the pin and proving against the data files would be wrong both ways: a
|
|
19
|
+
row referencing a hosted key the files lack would be refused, and a row colliding
|
|
20
|
+
with a hosted unique key would pass.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import shutil
|
|
26
|
+
import tempfile
|
|
27
|
+
from contextlib import contextmanager
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any, Iterable, Iterator
|
|
30
|
+
|
|
31
|
+
from .batch import ChunkSpec, build_batch, chunk_rows, validate_chunks
|
|
32
|
+
from .compiler import compile_bundle
|
|
33
|
+
from .validation import fail
|
|
34
|
+
|
|
35
|
+
Rows = dict[str, list[dict]]
|
|
36
|
+
#: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
|
|
37
|
+
SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
|
|
38
|
+
#: What a state-independent proof leaves to the platform batch.
|
|
39
|
+
DEFERRED = ("references", "unique")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@contextmanager
|
|
43
|
+
def proving_copy(root: Path) -> Iterator[Path]:
|
|
44
|
+
"""A scratch copy of the tree to prove rows against and throw away: build
|
|
45
|
+
outputs and captures left behind."""
|
|
46
|
+
with tempfile.TemporaryDirectory(prefix="gateway-prove-") as scratch:
|
|
47
|
+
staged = Path(scratch) / "world"
|
|
48
|
+
shutil.copytree(root, staged, ignore=shutil.ignore_patterns(*SCRATCH_IGNORE))
|
|
49
|
+
yield staged
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class RowProof:
|
|
53
|
+
"""One dry run over one tree. ``check`` is the batch lane's report (rows,
|
|
54
|
+
refusals, entity counts) over chunk files; ``run`` is the import door's report
|
|
55
|
+
over in-memory rows, refusing with a ``ContractError`` as the import would."""
|
|
56
|
+
|
|
57
|
+
proof: str
|
|
58
|
+
|
|
59
|
+
def __init__(self, root: Path):
|
|
60
|
+
self.root = root
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def of(cls, root: str | Path) -> "RowProof":
|
|
64
|
+
from .snapshot import current_default, pinned_default
|
|
65
|
+
|
|
66
|
+
root = Path(root)
|
|
67
|
+
pin = pinned_default(root)
|
|
68
|
+
if pin is None:
|
|
69
|
+
return FilesProof(root)
|
|
70
|
+
contract = compile_bundle(root, snapshots=False)
|
|
71
|
+
if current_default(root, contract) is not None:
|
|
72
|
+
return SnapshotProof(root, pin, contract)
|
|
73
|
+
return ContractProof(root, pin, contract)
|
|
74
|
+
|
|
75
|
+
def check(
|
|
76
|
+
self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
|
|
77
|
+
) -> dict:
|
|
78
|
+
raise NotImplementedError
|
|
79
|
+
|
|
80
|
+
def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
|
|
81
|
+
raise NotImplementedError
|
|
82
|
+
|
|
83
|
+
def _report(self, checked: dict) -> dict:
|
|
84
|
+
return {**checked, "proof": self.proof}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class FilesProof(RowProof):
|
|
88
|
+
proof = "files"
|
|
89
|
+
|
|
90
|
+
def check(
|
|
91
|
+
self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
|
|
92
|
+
) -> dict:
|
|
93
|
+
from .snapshot import EMPTY_DIGEST, build_default, default_snapshot
|
|
94
|
+
|
|
95
|
+
with proving_copy(self.root) as staged, tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
|
|
96
|
+
contract = compile_bundle(staged, snapshots=False)
|
|
97
|
+
build_default(staged, contract)
|
|
98
|
+
parent = Path(scratch) / "state.sqlite"
|
|
99
|
+
shutil.copyfile(default_snapshot(staged), parent)
|
|
100
|
+
return self._report(_batch(contract, parent, chunks, mode, merge, atomic, "check", EMPTY_DIGEST, redactor))
|
|
101
|
+
|
|
102
|
+
def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
|
|
103
|
+
from .data import import_rows
|
|
104
|
+
|
|
105
|
+
with proving_copy(self.root) as staged:
|
|
106
|
+
proven = import_rows(staged, rows, mode=mode, merge=merge, batch_id=batch_id, redact=redact)
|
|
107
|
+
return self._report({**proven, "changed": False})
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class _PinnedProof(RowProof):
|
|
111
|
+
def __init__(self, root: Path, pin: dict, contract: Any):
|
|
112
|
+
super().__init__(root)
|
|
113
|
+
self.pin = pin
|
|
114
|
+
self.contract = contract
|
|
115
|
+
|
|
116
|
+
def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
|
|
117
|
+
from .data import KIND_SCHEMA, _merged, _refuse_shapes
|
|
118
|
+
from .redaction import parse_mode, redact_rows, require_policy
|
|
119
|
+
|
|
120
|
+
redact = parse_mode(redact)
|
|
121
|
+
redacted = redact_rows(require_policy(self.root, redact), rows, redact)
|
|
122
|
+
settled = _merged(self.root, redacted, merge)
|
|
123
|
+
_refuse_shapes(self.root, settled)
|
|
124
|
+
with tempfile.TemporaryDirectory(prefix="gateway-prove-") as scratch:
|
|
125
|
+
chunks = [
|
|
126
|
+
ChunkSpec(chunk.ordinal, chunk.path, chunk.entity)
|
|
127
|
+
for chunk in chunk_rows(_tagged(settled), Path(scratch) / "chunks")
|
|
128
|
+
]
|
|
129
|
+
checked = self.check(chunks, mode=mode, merge=merge) if chunks else _empty()
|
|
130
|
+
_refuse(checked["refusals"], settled)
|
|
131
|
+
report = {"kind": KIND_SCHEMA, "batch": batch_id, "entity_counts": checked["entityCounts"], "changed": False}
|
|
132
|
+
if redact != "off":
|
|
133
|
+
report["redact"] = redact
|
|
134
|
+
return self._report({**report, **({"deferred": checked["deferred"]} if "deferred" in checked else {})})
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class SnapshotProof(_PinnedProof):
|
|
138
|
+
proof = "snapshot"
|
|
139
|
+
|
|
140
|
+
def check(
|
|
141
|
+
self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
|
|
142
|
+
) -> dict:
|
|
143
|
+
from .snapshot import default_snapshot
|
|
144
|
+
|
|
145
|
+
with tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
|
|
146
|
+
parent = Path(scratch) / "state.sqlite"
|
|
147
|
+
shutil.copyfile(default_snapshot(self.root), parent)
|
|
148
|
+
return self._report(
|
|
149
|
+
_batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin["batchDigest"], redactor)
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class ContractProof(_PinnedProof):
|
|
154
|
+
proof = "contract"
|
|
155
|
+
|
|
156
|
+
def check(
|
|
157
|
+
self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
|
|
158
|
+
) -> dict:
|
|
159
|
+
shaped = validate_chunks(self.contract, chunks, redactor=redactor)
|
|
160
|
+
counts = self.pin.get("entityCounts")
|
|
161
|
+
return self._report(
|
|
162
|
+
{
|
|
163
|
+
"rows": shaped["rows"],
|
|
164
|
+
"refusals": shaped["refusals"],
|
|
165
|
+
"refusalCount": len(shaped["refusals"]),
|
|
166
|
+
"entityCounts": dict(counts) if isinstance(counts, dict) else {},
|
|
167
|
+
"deferred": list(DEFERRED),
|
|
168
|
+
}
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _batch(contract, parent, chunks, mode, merge, atomic, batch_id, parent_digest, redactor) -> dict:
|
|
173
|
+
result = build_batch(
|
|
174
|
+
contract,
|
|
175
|
+
parent,
|
|
176
|
+
chunks,
|
|
177
|
+
mode=mode,
|
|
178
|
+
merge=merge,
|
|
179
|
+
atomic=atomic,
|
|
180
|
+
batch_id=batch_id,
|
|
181
|
+
parent_digest=parent_digest,
|
|
182
|
+
dry_run=True,
|
|
183
|
+
redactor=redactor,
|
|
184
|
+
)
|
|
185
|
+
return {
|
|
186
|
+
"rows": sum(1 for chunk in chunks for _ in _chunk_rows(chunk)),
|
|
187
|
+
"refusals": result["refusals"],
|
|
188
|
+
"refusalCount": result["refusalCount"],
|
|
189
|
+
"entityCounts": result["entityCounts"],
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _chunk_rows(chunk: ChunkSpec) -> Iterable[tuple[str, dict]]:
|
|
194
|
+
from .batch import iter_chunk_rows
|
|
195
|
+
|
|
196
|
+
return iter_chunk_rows(chunk.path, chunk.entity)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _tagged(rows: Rows) -> Iterator[tuple[str, dict]]:
|
|
200
|
+
for entity, listed in rows.items():
|
|
201
|
+
for row in listed:
|
|
202
|
+
yield entity, row
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _empty() -> dict:
|
|
206
|
+
return {"rows": 0, "refusals": [], "refusalCount": 0, "entityCounts": {}}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _refuse(refusals: list[dict], rows: Rows) -> None:
|
|
210
|
+
"""The state the rows collide with, as one ``ContractError`` carrying every
|
|
211
|
+
refusal (shape refusals were raised earlier, with their own codes)."""
|
|
212
|
+
if not refusals:
|
|
213
|
+
return
|
|
214
|
+
first = refusals[0]
|
|
215
|
+
total = sum(len(listed) for listed in rows.values())
|
|
216
|
+
summary = f"{len(refusals)} of {total} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
|
|
217
|
+
fail(
|
|
218
|
+
"state",
|
|
219
|
+
summary + first["detail"],
|
|
220
|
+
first["pointer"],
|
|
221
|
+
entity=first["entity"],
|
|
222
|
+
index=first["index"],
|
|
223
|
+
refused=len(refusals),
|
|
224
|
+
rows=total,
|
|
225
|
+
refusals=refusals,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
__all__ = [
|
|
230
|
+
"DEFERRED",
|
|
231
|
+
"SCRATCH_IGNORE",
|
|
232
|
+
"ContractProof",
|
|
233
|
+
"FilesProof",
|
|
234
|
+
"RowProof",
|
|
235
|
+
"SnapshotProof",
|
|
236
|
+
"proving_copy",
|
|
237
|
+
]
|