@zergai/ztc 0.4.0-beta.1 → 0.5.0-beta.1

package/dist/remote-broker/retained.py

@@ -0,0 +1,686 @@
+"""Offline confirmed restoration. Never disposes data or resolves mutations."""
+
+import argparse
+import contextlib
+import ctypes
+import errno
+import fcntl
+import json
+import logging
+import os
+import stat
+import sys
+from pathlib import Path
+from typing import Any, Iterator, Literal, Protocol
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Fixed adjacent operator library; isolated Python ignores ambient import paths.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from cutover import (  # noqa: E402
+    Authority,
+    FENCE,
+    Lease,
+    MAX_TOTAL,
+    Receipt,
+    UUID,
+    canonical,
+    digest,
+    directory,
+    entries,
+    json_data,
+    make_directory,
+    publish,
+    read_file,
+    sync_directory,
+)
+
+L = logging.getLogger(__name__)
+FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+
+
+class Manifest(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+    version: Literal[1]
+    clients_stopped: bool
+    authority: Authority
+    configured_staging_directory: str | None = None
+    receipt_id: str = Field(pattern=UUID.pattern)
+    destination: str = Field(min_length=1, max_length=4096)
+
+
+def physical(info: os.stat_result) -> list[int]:
+    return [info.st_dev, info.st_ino]
+
+
+def private(path: Path) -> None:
+    info = path.stat()
+    if info.st_uid != os.geteuid() or info.st_mode & 0o077:
+        raise ValueError(
+            "Operator state and staging must remain private to their owner"
+        )
+
+
+def relative(value: str) -> str:
+    if (
+        not value
+        or os.path.isabs(value)
+        or "\x00" in value
+        or os.path.normpath(value) != value
+        or any(part in ["", ".", ".."] for part in value.split("/"))
+    ):
+        raise ValueError(
+            "Destination must be a normalized relative path beneath the resource"
+        )
+    return value
+
+
+class OperatorAuthority(Protocol):
+    authority: Authority
+
+
+@contextlib.contextmanager
+def locked(manifest: OperatorAuthority) -> Iterator[None]:
+    state = directory(manifest.authority.state_directory)
+    private(state)
+    descriptor = os.open(state / "lock", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+    try:
+        if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+            raise ValueError("Missing regular broker lock evidence")
+        fcntl.flock(descriptor, fcntl.LOCK_EX)
+        yield
+    finally:
+        os.close(descriptor)
+
+
+@contextlib.contextmanager
+def parent(
+    root: Path, path: str, root_identity: list[int]
+) -> Iterator[tuple[int, str]]:
+    """Hold every component; a substituted symlink is never followed."""
+    with contextlib.ExitStack() as handles:
+        descriptor = os.open(root, FLAGS)
+        handles.callback(os.close, descriptor)
+        if physical(os.fstat(descriptor)) != root_identity:
+            raise ValueError("Resource root identity changed")
+        parts = relative(path).split("/")
+        for part in parts[:-1]:
+            descriptor = os.open(part, FLAGS, dir_fd=descriptor)
+            handles.callback(os.close, descriptor)
+        yield descriptor, parts[-1]
+
+
+def observe(descriptor: int, leaf: str) -> dict[str, Any]:
+    try:
+        info = os.stat(leaf, dir_fd=descriptor, follow_symlinks=False)
+    except FileNotFoundError:
+        return {"exists": False}
+    value: dict[str, Any] = {
+        "exists": True,
+        "identity": physical(info),
+        "mode": info.st_mode,
+        "size": info.st_size,
+        "mtime_ns": info.st_mtime_ns,
+        "ctime_ns": info.st_ctime_ns,
+        "links": info.st_nlink,
+    }
+    if stat.S_ISREG(info.st_mode):
+        file = os.open(
+            leaf, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=descriptor
+        )
+        try:
+            opened = os.fstat(file)
+            if physical(opened) != physical(info) or not stat.S_ISREG(opened.st_mode):
+                raise ValueError("Retained object changed during observation")
+            with os.fdopen(file, "rb", closefd=False) as stream:
+                value["sample_sha256"] = digest(stream.read(1048576))
+        finally:
+            os.close(file)
+    elif stat.S_ISDIR(info.st_mode):
+        child = os.open(leaf, FLAGS, dir_fd=descriptor)
+        try:
+            if physical(os.fstat(child)) != physical(info):
+                raise ValueError("Retained directory changed during observation")
+            value["empty"] = not os.listdir(child)
+        finally:
+            os.close(child)
+    elif stat.S_ISLNK(info.st_mode):
+        value["link_target"] = os.readlink(leaf, dir_fd=descriptor)
+    else:
+        raise ValueError("Retained object must be a file, directory or symbolic link")
+    after = os.stat(leaf, dir_fd=descriptor, follow_symlinks=False)
+    # Reading evidence may update atime; it is not an object mutation.
+    if (
+        physical(after) != physical(info)
+        or after.st_mode != info.st_mode
+        or after.st_size != info.st_size
+        or after.st_mtime_ns != info.st_mtime_ns
+        or after.st_ctime_ns != info.st_ctime_ns
+        or after.st_nlink != info.st_nlink
+    ):
+        raise ValueError("Retained object changed during observation")
+    return value
+
+
+def same_object(observed: dict[str, Any], recorded: dict[str, Any]) -> bool:
+    # Namespace rename changes ctime. All other reviewed metadata remains bound.
+    return {k: v for k, v in observed.items() if k != "ctime_ns"} == {
+        k: v for k, v in recorded.items() if k != "ctime_ns"
+    }
+
+
+def rename_no_replace(source: int, destination: int, leaf: str) -> None:
+    library = ctypes.CDLL(None, use_errno=True)
+    if sys.platform == "linux":
+        function, flag = library.renameat2, 1
+    elif sys.platform == "darwin":
+        function, flag = library.renameatx_np, 4
+    else:
+        raise OSError(
+            errno.ENOTSUP, "Restoration requires atomic no-replace rename support"
+        )
+    function.argtypes = [
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_uint,
+    ]
+    function.restype = ctypes.c_int
+    if function(source, b"target", destination, os.fsencode(leaf), flag) != 0:
+        error = ctypes.get_errno()
+        raise OSError(
+            error,
+            "Restoration refused occupied or changed destination: "
+            + os.strerror(error),
+        )
+
+
+class Survey:
+    def __init__(self, manifest: Manifest, reference: Path) -> None:
+        self.manifest = manifest
+        self.state = directory(manifest.authority.state_directory)
+        self.staging = directory(manifest.authority.staging_directory)
+        private(self.state)
+        private(self.staging)
+        self.files: dict[str, bytes] = {}
+        self.blockers: list[str] = []
+        self.inventory: list[dict[str, Any]] = []
+        self.roots: dict[str, Path] = {}
+        self.receipts: dict[str, dict[str, Any]] = {}
+        self.bindings: list[dict[str, Any]] = []
+        self.reference = read_file(reference)
+        if (
+            b"os.path.lexists(os.path.join(control, 'cutover-fence.json'))"
+            not in self.reference
+        ):
+            raise ValueError("Reviewed worker lacks the operator freeze protocol")
+        if read_file(Path(manifest.authority.worker_file)) != self.reference:
+            raise ValueError(
+                "Deployed worker differs from the reviewed fence-capable worker"
+            )
+
+    def capture(self, path: Path, name: str) -> bytes:
+        data = read_file(path)
+        self.files[name] = data
+        if sum(map(len, self.files.values())) > MAX_TOTAL:
+            raise ValueError("Recovery evidence exceeds the aggregate byte bound")
+        return data
+
+    def inspect(self) -> dict[str, Any]:
+        manifest = self.manifest
+        if not manifest.clients_stopped:
+            self.blockers.append("Operator must stop all clients before recovery")
+        resources = {r.id: r for r in manifest.authority.resources}
+        if len(resources) != len(manifest.authority.resources):
+            raise ValueError("Resource identities must be unique")
+        identities: dict[str, list[int]] = {}
+        for resource in manifest.authority.resources:
+            root = directory(resource.root_directory)
+            relative(manifest.destination)
+            if (
+                not os.path.isabs(resource.configured_root)
+                or os.path.normpath(resource.configured_root)
+                != resource.configured_root
+            ):
+                raise ValueError("Configured resource root must be canonical")
+            if resource.configured_root in identities or any(
+                root.is_relative_to(p) or p.is_relative_to(root)
+                for p in [self.state, self.staging]
+            ):
+                raise ValueError(
+                    "Resource roots must be unique and outside operator evidence"
+                )
+            self.roots[resource.id] = root
+            identities[resource.configured_root] = physical(root.stat())
+            self.bindings.append(
+                {**resource.model_dump(), "root_identity": physical(root.stat())}
+            )
+        leases = json_data(self.capture(self.state / "leases.json", "leases.json"))
+        if not isinstance(leases, dict):
+            raise ValueError("Invalid lease registry evidence")
+        for key, data in leases.items():
+            lease = Lease.model_validate(data)
+            if (
+                key != digest(lease.root.encode())
+                or identities.get(lease.root) != lease.identity
+            ):
+                raise ValueError(
+                    "Lease root identity lacks its original physical resource"
+                )
+            if lease.owner_id is not None:
+                self.blockers.append(
+                    "Stop every recorded lease before recovery, including expired owners"
+                )
+        for path in entries(directory(str(self.state / "receipts"))):
+            raw = self.capture(path, "receipts/" + path.name)
+            validated = Receipt.model_validate_json(raw)
+            mapped_resource = resources.get(validated.resource_id)
+            if (
+                validated.id != path.stem
+                or mapped_resource is None
+                or validated.configured_root != mapped_resource.configured_root
+                or validated.endpoint != mapped_resource.endpoint
+                or validated.root_identity
+                != identities[mapped_resource.configured_root]
+                or validated.target.get("path")
+                != os.path.join(
+                    mapped_resource.configured_root, relative(validated.requested_path)
+                )
+            ):
+                raise ValueError(
+                    "Receipt binding or root identity differs from the original resource"
+                )
+            self.receipts[validated.id] = json_data(raw)
+            has_stage = "deletion_staging" in self.receipts[validated.id]
+            retained, stage_identity, directory_state = (
+                self.stage(validated.id)
+                if has_stage
+                else ({"exists": False}, None, "no_operation_directory")
+            )
+            self.inventory.append(
+                {
+                    "receipt_id": validated.id,
+                    "retained": retained["exists"],
+                    "resource_id": validated.resource_id,
+                    "root_identity": validated.root_identity,
+                    "endpoint": validated.endpoint.model_dump(),
+                    "original_target": validated.target,
+                    "directory_identity": stage_identity,
+                    "directory_state": directory_state,
+                    "captured_identity": retained.get("identity"),
+                    "recorded_captured_identity": (
+                        self.receipts[validated.id]["deletion_staging"].get(
+                            "captured_identity"
+                        )
+                        if has_stage
+                        else None
+                    ),
+                }
+            )
+        if manifest.receipt_id not in self.receipts:
+            raise ValueError("Original retained deletion receipt is missing")
+        receipt = self.receipts[manifest.receipt_id]
+        if receipt["operation"] != "delete" or "deletion_staging" not in receipt:
+            raise ValueError("Selected receipt has no retained deletion capture")
+        source, stage_identity, directory_state = self.stage(manifest.receipt_id)
+        if not source["exists"]:
+            self.blockers.append(
+                "Retained object is absent; absence proves no deletion or reclaimed quota"
+            )
+        root = self.roots[receipt["resource_id"]]
+        with parent(root, manifest.destination, receipt["root_identity"]) as (
+            descriptor,
+            leaf,
+        ):
+            destination = observe(descriptor, leaf)
+            destination_parent_identity = physical(os.fstat(descriptor))
+        if destination["exists"]:
+            self.blockers.append(
+                "Restoration destination is occupied; choose an empty destination"
+            )
+        selected = {
+            "receipt_id": receipt["id"],
+            "resource_id": receipt["resource_id"],
+            "endpoint": receipt["endpoint"],
+            "root_identity": receipt["root_identity"],
+            "original_target": receipt["target"],
+            "tool_call_id": receipt["tool_call_id"],
+            "recorded_captured_identity": receipt["deletion_staging"].get(
+                "captured_identity"
+            ),
+            "captured_identity": source.get("identity"),
+            "directory_identity": stage_identity,
+            "directory_state": directory_state,
+            "staging_identity": physical(self.staging.stat()),
+            "state_identity": physical(self.state.stat()),
+            "links": source.get("links"),
+            "source": source,
+            "destination": str(root / manifest.destination),
+            "current_destination": destination,
+            "destination_parent_identity": destination_parent_identity,
+        }
+        plan = {
+            "version": 1,
+            "manifest": manifest.model_dump(),
+            "bindings": sorted(self.bindings, key=lambda r: r["id"]),
+            "worker_sha256": digest(self.reference),
+            "files": {n: digest(data) for n, data in sorted(self.files.items())},
+            "selected": selected,
+        }
+        return {
+            "plan": plan,
+            "preview": {
+                "ready": not self.blockers,
+                "digest": digest(canonical(plan)),
+                "blockers": self.blockers,
+                "inventory": self.inventory,
+                "selected": selected,
+            },
+        }
+
+    def stage(self, receipt_id: str) -> tuple[dict[str, Any], list[int], str]:
+        receipt = self.receipts[receipt_id]
+        stage = receipt["deletion_staging"]
+        configured = self.manifest.configured_staging_directory or str(self.staging)
+        if (
+            not os.path.isabs(configured)
+            or os.path.normpath(configured) != configured
+            or stage.get("path") != os.path.join(configured, receipt_id, "target")
+        ):
+            raise ValueError("Retained path lacks its exact private receipt binding")
+        try:
+            path = directory(str(self.staging / receipt_id))
+        except FileNotFoundError:
+            from retained_cleanup import completed_evidence
+
+            if completed_evidence(self.state, self.staging, receipt):
+                return (
+                    {"exists": False},
+                    stage["directory_identity"],
+                    "reviewed_empty_directory_removed",
+                )
+            if stage.get("state") in ["removed", "restored"]:
+                return (
+                    {"exists": False},
+                    stage["directory_identity"],
+                    "absent_operation_directory",
+                )
+            raise ValueError(
+                "Private retained directory missing; explicit recovery required"
+            )
+        private(path)
+        descriptor = os.open(path, FLAGS)
+        try:
+            identity = physical(os.fstat(descriptor))
+            if identity != stage.get("directory_identity"):
+                raise ValueError("Private directory identity changed")
+            observed = observe(descriptor, "target")
+            if (
+                observed["exists"]
+                and stage.get("captured_identity") is not None
+                and (
+                    observed["identity"] != stage["captured_identity"]
+                    or observed["mode"] != stage.get("captured_mode")
+                )
+            ):
+                raise ValueError("Recorded captured object identity changed")
+            directory_state = (
+                "contains_retained_target"
+                if observed["exists"]
+                else (
+                    "nonempty_operation_directory"
+                    if os.listdir(descriptor)
+                    else "empty_operation_directory"
+                )
+            )
+            return observed, identity, directory_state
+        finally:
+            os.close(descriptor)
+
+    @contextlib.contextmanager
+    def handles(self, plan: dict[str, Any]) -> Iterator[tuple[int, int, str]]:
+        selected = plan["selected"]
+        receipt = self.receipts[self.manifest.receipt_id]
+        with contextlib.ExitStack() as stack:
+            source = os.open(self.staging / receipt["id"], FLAGS)
+            stack.callback(os.close, source)
+            if physical(os.fstat(source)) != selected["directory_identity"]:
+                raise ValueError("Private directory identity changed")
+            destination, leaf = stack.enter_context(
+                parent(
+                    self.roots[receipt["resource_id"]],
+                    self.manifest.destination,
+                    selected["root_identity"],
+                )
+            )
+            if (
+                physical(os.fstat(destination))
+                != selected["destination_parent_identity"]
+            ):
+                raise ValueError("Destination parent identity changed")
+            yield source, destination, leaf
+
+
+def valid_digest(value: str | None) -> str:
+    if (
+        value is None
+        or len(value) != 64
+        or any(c not in "0123456789abcdef" for c in value)
+    ):
+        raise ValueError("Explicit current preview digest confirmation is required")
+    return value
+
+
+def check_original(current: dict[str, Any], plan: dict[str, Any]) -> None:
+    if any(
+        current[k] != plan[k]
+        for k in ["manifest", "bindings", "worker_sha256", "files"]
+    ):
+        raise ValueError(
+            "Recovery original evidence or root binding changed; retain the fence"
+        )
+    for key in [
+        "directory_identity",
+        "staging_identity",
+        "state_identity",
+        "destination_parent_identity",
+    ]:
+        if current["selected"][key] != plan["selected"][key]:
+            raise ValueError("Recovery directory identity changed; retain the fence")
+
+
+def restored_record(plan: dict[str, Any], recovery_digest: str) -> dict[str, Any]:
+    selected = plan["selected"]
+    return {
+        "version": 1,
+        "digest": recovery_digest,
+        "decision": "restore_retained_object",
+        "receipt_id": selected["receipt_id"],
+        "captured_identity": selected["captured_identity"],
+        "destination": selected["destination"],
+        "original_receipt_sha256": plan["files"][
+            "receipts/" + selected["receipt_id"] + ".json"
+        ],
+        "earlier_mutation_resolved": False,
+        "quota_reclaimed": False,
+    }
+
+
+def completion(
+    survey: Survey, plan: dict[str, Any], recovery_digest: str
+) -> dict[str, Any]:
+    with survey.handles(plan) as (source, destination, leaf):
+        if observe(source, "target")["exists"]:
+            raise ValueError(
+                "Recovery source still exists; restoration is not complete"
+            )
+        observed = observe(destination, leaf)
+        if not same_object(observed, plan["selected"]["source"]):
+            raise ValueError(
+                "Restored destination changed; explicit operator recovery required"
+            )
+        # Settle a visible rename whose previous response was lost.
+        os.fsync(source)
+        os.fsync(destination)
+    return {
+        "digest": recovery_digest,
+        "restored": restored_record(plan, recovery_digest),
+        "current_destination": observed,
+    }
+
+
+def execute(
+    operation: str,
+    manifest: Manifest,
+    reference: Path,
+    confirmation: str | None,
+    recovery_digest: str | None,
+) -> dict[str, Any]:
+    with locked(manifest):
+        survey = Survey(manifest, reference)
+        current = survey.inspect()
+        state = survey.state
+        if operation == "preview":
+            if os.path.lexists(state / FENCE):
+                current["preview"]["ready"] = False
+                current["preview"]["blockers"].append(
+                    "Authority is already frozen; recover its existing operator evidence"
+                )
+            return {"preview": current["preview"]}
+        confirmed = valid_digest(confirmation)
+        selected_digest = (
+            valid_digest(recovery_digest) if operation == "release" else confirmed
+        )
+        archive = state / "retained-recovery" / selected_digest
+        marker = canonical(
+            {"version": 1, "kind": "retained-recovery", "digest": selected_digest}
+        )
+        plan_path = archive / "plan.json"
+        if os.path.lexists(plan_path):
+            plan = json_data(read_file(plan_path))
+            if digest(canonical(plan)) != selected_digest:
+                raise ValueError("Recovery plan damaged; preserve the authority fence")
+            check_original(current["plan"], plan)
+            original = read_file(archive / "original-receipt.json")
+            if (
+                digest(original)
+                != plan["files"]["receipts/" + manifest.receipt_id + ".json"]
+            ):
+                raise ValueError(
+                    "Archived original receipt damaged; preserve the authority fence"
+                )
+        else:
+            plan = current["plan"]
+            if selected_digest != current["preview"]["digest"]:
+                raise ValueError(
+                    "Preview confirmation changed; review current retained evidence"
+                )
+        if not manifest.clients_stopped or any("lease" in b for b in survey.blockers):
+            raise ValueError(
+                "Recovery requires stopped clients and every recorded lease"
+            )
+        if operation == "release":
+            if json_data(read_file(archive / "restored.json")) != restored_record(
+                plan, selected_digest
+            ):
+                raise ValueError("Recovery has no matching durable restoration receipt")
+            review = completion(survey, plan, selected_digest)
+            if confirmed != digest(canonical(review)):
+                raise ValueError(
+                    "Release observation changed; review restored state before confirming"
+                )
+            released = canonical(
+                {
+                    "version": 1,
+                    "digest": selected_digest,
+                    "review_token": confirmed,
+                    "state": "released",
+                }
+            )
+            if os.path.lexists(state / FENCE):
+                if read_file(state / FENCE) != marker:
+                    raise ValueError(
+                        "Operator fence conflicts; preserve it for recovery"
+                    )
+                publish(archive / "released.json", released)
+                os.unlink(state / FENCE)
+            elif read_file(archive / "released.json") != released:
+                raise ValueError(
+                    "Missing matching release evidence; preserve current state"
+                )
+            sync_directory(state)
+            return {
+                "state": "released",
+                "digest": selected_digest,
+                "archive": str(archive),
+                "earlier_mutation_resolved": False,
+            }
+        if (
+            not plan["selected"]["source"]["exists"]
+            or plan["selected"]["current_destination"]["exists"]
+        ):
+            raise ValueError(
+                "Restore requires a reviewed retained object and empty destination"
+            )
+        publish(state / FENCE, marker)
+        make_directory(archive)
+        publish(
+            archive / "original-receipt.json",
+            survey.files["receipts/" + manifest.receipt_id + ".json"],
+        )
+        publish(plan_path, canonical(plan))
+        with survey.handles(plan) as (source, destination, leaf):
+            observed = observe(source, "target")
+            if observed["exists"]:
+                if not same_object(observed, plan["selected"]["source"]):
+                    raise ValueError(
+                        "Retained object changed; preserve the authority fence"
+                    )
+                rename_no_replace(source, destination, leaf)
+                os.fsync(source)
+                os.fsync(destination)
+        review = completion(survey, plan, selected_digest)
+        # Namespace descriptors must still map to the reviewed named directories.
+        check_original(Survey(manifest, reference).inspect()["plan"], plan)
+        publish(
+            archive / "restored.json", canonical(restored_record(plan, selected_digest))
+        )
+        return {
+            "state": "restored",
+            "digest": selected_digest,
+            "archive": str(archive),
+            "review_token": digest(canonical(review)),
+            "review": review,
+            "earlier_mutation_resolved": False,
+        }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Preview and confirm retained deletion restoration"
+    )
+    parser.add_argument("operation", choices=["preview", "restore", "release"])
+    parser.add_argument("--manifest", required=True)
+    parser.add_argument("--confirm")
+    parser.add_argument("--recovery-digest")
+    parser.add_argument("--worker-file")
+    args = parser.parse_args()
+    try:
+        manifest = Manifest.model_validate_json(read_file(Path(args.manifest)))
+        reference = (
+            Path(args.worker_file)
+            if args.worker_file
+            else Path(__file__).with_name("worker.py")
+        )
+        result = execute(
+            args.operation, manifest, reference, args.confirm, args.recovery_digest
+        )
+        print(json.dumps({"ok": True, **result}, allow_nan=False))
+    except (ValueError, OSError) as error:
+        print(json.dumps({"ok": False, "error": str(error)}))
+        raise SystemExit(2) from error
+
+
+if __name__ == "__main__":
+    main()