create-egregore 0.20.2 → 0.21.0-runtime-mvp.1

package/runtime/prime/egregore_runtime/upgrade.py

@@ -0,0 +1,770 @@
+"""Transactional Runtime upgrade engine — staged, durable, per instance.
+
+The engine prepares a candidate Runtime for exactly one registered Egregore
+instance while the currently active Runtime keeps serving. Preparation is a
+sequence of durable stages; every transition is an atomic write to an
+instance-scoped state file, so an interrupted worker resumes completed work
+instead of restarting it. Activation is a separate, explicit, atomic step
+that only succeeds when every gate passes — BM25 alone never activates.
+
+All state lives under the instance's own runtime root, keyed by the same
+stable instance hash the QMD adapter uses. Nothing here reads or writes
+another instance's files, processes, indexes, or configuration.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Mapping
+
+from .adapters.qmd import QMD_VERSION, RERANKING_ENABLED, QmdLocalRetriever
+
+SCHEMA_VERSION = "egregore-runtime-upgrade/v1"
+
+# Stage keys in preparation order. Labels are the user-facing truth shown by
+# Settings; keep them in plain team language.
+STAGES: tuple[tuple[str, str], ...] = (
+    ("sync", "Team memory synchronized"),
+    ("candidate", "Runtime candidate recorded"),
+    ("install", "Runtime candidate installed"),
+    ("node", "Node runtime ready"),
+    ("qmd", "QMD installed"),
+    ("bm25", "Keyword index built"),
+    ("embed", "Meaning index generated"),
+    ("delta", "Late team changes embedded"),
+    ("verify", "Retrieval, identity, and boundary verified"),
+)
+
+# The harness bundle applied to the instance at activation. All four bundles
+# must be present in the candidate package; the Claude bundle carries the
+# shared core (bin/, egregore_runtime/, .claude/) this instance runs on.
+BUNDLE_TREES = ("claude", "codex", "pi", "prime")
+# Activation applies every harness adapter tree, not only Claude's: the
+# Codex black-box found a live instance whose .codex adapters were still the
+# pre-cutover generation, sending Codex sessions through a broken global QMD
+# and graph-era instructions while Claude ran the new Runtime. Shared paths
+# (bin/, egregore_runtime/) are byte-identical across trees; the first tree
+# wins and duplicates are skipped.
+APPLY_TREES = ("claude", "codex", "pi", "prime")
+APPLY_TREE = APPLY_TREES[0]
+
+STATUS_PENDING = "pending"
+STATUS_PREPARING = "preparing"
+STATUS_FAILED = "failed"
+STATUS_READY = "ready-to-activate"
+STATUS_ACTIVE = "active"
+STATUS_CANCELLED = "cancelled"
+STATUS_ROLLED_BACK = "rolled-back"
+
+
+class UpgradeError(RuntimeError):
+    pass
+
+
+def _now() -> int:
+    return int(time.time())
+
+
+@dataclass
+class UpgradeStore:
+    """Durable, instance-scoped upgrade state with atomic writes."""
+
+    root: Path
+
+    @property
+    def state_path(self) -> Path:
+        return self.root / "state.json"
+
+    @property
+    def activation_path(self) -> Path:
+        return self.root / "active.json"
+
+    @property
+    def cancel_path(self) -> Path:
+        return self.root / "cancel"
+
+    def load(self) -> dict[str, Any] | None:
+        try:
+            value = json.loads(self.state_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return None
+        return value if isinstance(value, dict) else None
+
+    def write(self, state: Mapping[str, Any]) -> None:
+        self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+        os.chmod(self.root, 0o700)
+        payload = dict(state)
+        payload["updated_at"] = _now()
+        temporary = self.state_path.with_suffix(f".tmp.{os.getpid()}")
+        temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        os.chmod(temporary, 0o600)
+        temporary.replace(self.state_path)
+
+    def load_activation(self) -> dict[str, Any] | None:
+        try:
+            value = json.loads(self.activation_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return None
+        return value if isinstance(value, dict) else None
+
+    def write_activation(self, record: Mapping[str, Any]) -> None:
+        self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+        temporary = self.activation_path.with_suffix(f".tmp.{os.getpid()}")
+        temporary.write_text(json.dumps(dict(record), indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        os.chmod(temporary, 0o600)
+        temporary.replace(self.activation_path)
+
+    def cancel_requested(self) -> bool:
+        return self.cancel_path.exists()
+
+    def request_cancel(self) -> None:
+        self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+        self.cancel_path.write_text(str(_now()) + "\n", encoding="utf-8")
+
+    def clear_cancel(self) -> None:
+        self.cancel_path.unlink(missing_ok=True)
+
+
+def installation_root(repository_root: Path) -> Path:
+    """The main installation directory a checkout belongs to.
+
+    A linked git worktree (``.git`` is a file) belongs to its main checkout;
+    upgrade state is per installation, so every worktree of one install
+    shares it. A standalone clone is its own installation.
+    """
+    root = repository_root.resolve()
+    git_marker = root / ".git"
+    if git_marker.is_file():
+        try:
+            content = git_marker.read_text(encoding="utf-8")
+            for line in content.splitlines():
+                if line.startswith("gitdir:"):
+                    git_dir = Path(line.split(":", 1)[1].strip())
+                    if not git_dir.is_absolute():
+                        git_dir = (root / git_dir).resolve()
+                    # <main>/.git/worktrees/<name> → <main>
+                    if git_dir.parent.name == "worktrees" and git_dir.parent.parent.name == ".git":
+                        return git_dir.parent.parent.parent.resolve()
+        except OSError:
+            pass
+    return root
+
+
+def instance_key(repository_root: Path) -> str:
+    """Durable upgrade-state key: stable org identity × installation identity.
+
+    Identity is never inferred from a path alone: the stable organization ID
+    from egregore.json is required and leads the key, so two organizations
+    that share, symlink, or otherwise resolve to the same memory repository
+    can never share activation, worker, lock, or rollback state. The
+    installation path is composed in explicitly so two installs of the same
+    organization stay independent, while worktrees of one install share it.
+    A checkout without a stable org_id fails closed.
+    """
+    root = repository_root.resolve()
+    try:
+        config = json.loads((root / "egregore.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise UpgradeError(f"cannot read egregore.json for instance identity: {exc}") from exc
+    org_id = str(config.get("org_id") or "").strip()
+    if not org_id:
+        raise UpgradeError(
+            "this instance has no stable org_id; upgrade state fails closed "
+            "without a stable organization identity"
+        )
+    install = installation_root(root)
+    import hashlib
+
+    return hashlib.sha256(f"{org_id}|{install}".encode("utf-8")).hexdigest()[:16]
+
+
+def upgrade_root(repository_root: Path) -> Path:
+    """Instance-scoped upgrade state dir, keyed by composed stable identity."""
+    override = os.environ.get("EGREGORE_UPGRADE_ROOT")
+    if override:
+        return Path(override).expanduser() / instance_key(repository_root)
+    return Path.home() / ".egregore" / "runtime" / "upgrade" / instance_key(repository_root)
+
+
+def build_cli_context(repository_root: Path):
+    """Store/engine/retriever wiring for the CLI — keeps retrieval adapter
+    imports out of harness_cli, which must stay behind Runtime contracts."""
+    from .runtime import local_runtime
+
+    retriever = QmdLocalRetriever(repository_root=repository_root)
+    store = UpgradeStore(root=upgrade_root(repository_root))
+    engine = UpgradeEngine(
+        repository_root=repository_root,
+        store=store,
+        runtime=local_runtime(repository_root, push_remote=False),
+        retriever=retriever,
+    )
+    return store, engine, retriever
+
+
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+        return True
+    except (OSError, ValueError, TypeError):
+        return False
+
+
+def new_state(
+    *,
+    instance_path: Path,
+    org_id: str,
+    instance_hash: str,
+    candidate_version: str,
+    active_version: str,
+    candidate_source: str = "",
+) -> dict[str, Any]:
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "instance_path": str(instance_path),
+        "org_id": org_id,
+        "instance_hash": instance_hash,
+        "candidate_version": candidate_version,
+        "candidate_source": candidate_source,
+        "candidate_digest": None,
+        "installed_path": None,
+        "active_version": active_version,
+        "rollback_version": active_version,
+        "status": STATUS_PENDING,
+        "stages": [
+            {"key": key, "label": label, "state": "pending", "detail": ""}
+            for key, label in STAGES
+        ],
+        "source_revision": None,
+        "semantic_revision": None,
+        "progress": {},
+        "gates": {},
+        "failure": None,
+        "worker": None,
+        "started_at": _now(),
+    }
+
+
+def stage_index(state: Mapping[str, Any], key: str) -> int:
+    for index, stage in enumerate(state.get("stages", ())):
+        if stage.get("key") == key:
+            return index
+    raise UpgradeError(f"unknown upgrade stage: {key}")
+
+
+def describe(state: Mapping[str, Any] | None, *, worker_alive: bool | None = None) -> dict[str, Any]:
+    """Status projection for adapters (Settings, session start, readiness)."""
+    if state is None:
+        return {"schema_version": SCHEMA_VERSION, "status": "none"}
+    projected = dict(state)
+    worker = state.get("worker") or {}
+    pid = worker.get("pid")
+    alive = worker_alive if worker_alive is not None else (_pid_alive(pid) if pid else False)
+    projected["worker_alive"] = bool(alive)
+    # A preparing status with a dead worker is not "active work" — it is a
+    # resumable interruption and must be presented as such.
+    if state.get("status") == STATUS_PREPARING and not alive:
+        projected["status"] = "interrupted"
+    return projected
+
+
+@dataclass
+class UpgradeEngine:
+    """Runs preparation stages against one instance. Resumable by design."""
+
+    repository_root: Path
+    store: UpgradeStore
+    runtime: Any  # EgregoreRuntime; typed loosely to keep import edges thin
+    retriever: QmdLocalRetriever
+    poll_seconds: float = 1.0
+    log: Any = None  # callable(str) for worker logs; never user-facing
+
+    def _mark(self, state: dict[str, Any], key: str, stage_state: str, detail: str = "") -> None:
+        index = stage_index(state, key)
+        stage = state["stages"][index]
+        stage["state"] = stage_state
+        stage["detail"] = detail
+        if stage_state == "running" and "started_at" not in stage:
+            stage["started_at"] = _now()
+        if stage_state in ("done", "failed"):
+            stage["finished_at"] = _now()
+        self.store.write(state)
+
+    def _fail(self, state: dict[str, Any], key: str, message: str) -> None:
+        self._mark(state, key, "failed", message[:300])
+        state["status"] = STATUS_FAILED
+        state["failure"] = {"stage": key, "message": message[:300]}
+        self.store.write(state)
+
+    def _cancelled(self, state: dict[str, Any]) -> bool:
+        if not self.store.cancel_requested():
+            return False
+        state["status"] = STATUS_CANCELLED
+        self.store.write(state)
+        return True
+
+    def _stage_done(self, state: dict[str, Any], key: str) -> bool:
+        return state["stages"][stage_index(state, key)].get("state") == "done"
+
+    # -- stages ----------------------------------------------------------
+
+    def _run_sync(self, state: dict[str, Any]) -> bool:
+        receipt = self.runtime.synchronize()
+        current = bool(getattr(receipt, "canonical_current", False))
+        if not current:
+            failures = tuple(getattr(receipt, "failures", ()) or ())
+            self._fail(state, "sync", "; ".join(failures) or "team memory could not be synchronized")
+            return False
+        self._mark(state, "sync", "done", "canonical memory current")
+        return True
+
+    def _run_candidate(self, state: dict[str, Any]) -> bool:
+        revision = self.retriever._source_revision()
+        state["source_revision"] = revision
+        self._mark(state, "candidate", "done", revision)
+        return True
+
+    @staticmethod
+    def _sha256(path: Path) -> str:
+        import hashlib
+
+        digest = hashlib.sha256()
+        with path.open("rb") as handle:
+            for block in iter(lambda: handle.read(1 << 20), b""):
+                digest.update(block)
+        return digest.hexdigest()
+
+    def _run_install(self, state: dict[str, Any]) -> bool:
+        import shutil
+        import tarfile
+
+        source = Path(state.get("candidate_source") or "")
+        if not source.is_file():
+            self._fail(state, "install", "no candidate package tarball was staged for this upgrade")
+            return False
+        state["candidate_digest"] = self._sha256(source)
+        self.store.write(state)
+
+        target = self.store.root / "candidate"
+        if target.exists():
+            shutil.rmtree(target)
+        target.mkdir(parents=True, mode=0o700)
+        try:
+            with tarfile.open(source, "r:gz") as archive:
+                for member in archive.getmembers():
+                    member_path = (target / member.name).resolve()
+                    if not str(member_path).startswith(str(target.resolve())):
+                        raise UpgradeError(f"unsafe path in candidate archive: {member.name}")
+                archive.extractall(target)
+        except Exception as exc:
+            self._fail(state, "install", f"candidate extraction failed: {exc}")
+            return False
+
+        package_root = target / "package"
+        try:
+            manifest = json.loads((package_root / "package.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError) as exc:
+            self._fail(state, "install", f"candidate package.json unreadable: {exc}")
+            return False
+        if str(manifest.get("version")) != state["candidate_version"]:
+            self._fail(
+                state,
+                "install",
+                f"candidate package is {manifest.get('version')}, expected {state['candidate_version']}",
+            )
+            return False
+        scripts = manifest.get("scripts") or {}
+        for lifecycle in ("preinstall", "install", "postinstall"):
+            if lifecycle in scripts:
+                self._fail(state, "install", f"candidate declares a {lifecycle} lifecycle script; refused")
+                return False
+        missing = [tree for tree in BUNDLE_TREES if not (package_root / "runtime" / tree).is_dir()]
+        if missing:
+            self._fail(state, "install", "candidate is missing packaged bundles: " + ", ".join(missing))
+            return False
+        state["installed_path"] = str(package_root)
+        self._mark(
+            state,
+            "install",
+            "done",
+            f"{manifest.get('version')} · sha256 {state['candidate_digest'][:16]} · all four bundles present",
+        )
+        return True
+
+    def _run_node(self, state: dict[str, Any]) -> bool:
+        node = self.retriever.environment.get("EGREGORE_QMD_NODE_BIN")
+        if not node:
+            self._fail(state, "node", "no usable Node runtime was found for this instance")
+            return False
+        probe = subprocess.run(
+            (node, "--version"), capture_output=True, text=True, timeout=15, check=False
+        )
+        if probe.returncode != 0:
+            self._fail(state, "node", "the selected Node runtime failed its version probe")
+            return False
+        self._mark(state, "node", "done", f"{probe.stdout.strip()} at instance-selected path")
+        return True
+
+    def _run_qmd(self, state: dict[str, Any]) -> bool:
+        # An explicitly pinned QMD (constructor command or EGREGORE_QMD_BIN)
+        # is the instance's chosen binary — verify it instead of installing
+        # over it. Otherwise ensure the instance-owned managed install.
+        if self.retriever._configured_command or self.retriever.environment.get("EGREGORE_QMD_BIN"):
+            try:
+                self.retriever._ensure_version()
+            except Exception as exc:
+                self._fail(state, "qmd", f"pinned QMD failed verification: {exc}")
+                return False
+            self._mark(state, "qmd", "done", f"QMD {QMD_VERSION} (pinned by configuration)")
+            return True
+        binary = self.retriever._managed_binary()
+        if not (binary.is_file() and os.access(binary, os.X_OK)):
+            try:
+                binary = self.retriever.install_managed()
+            except Exception as exc:
+                self._fail(state, "qmd", f"QMD {QMD_VERSION} install failed: {exc}")
+                return False
+        self._mark(state, "qmd", "done", f"QMD {QMD_VERSION} (instance-owned)")
+        return True
+
+    def _run_bm25(self, state: dict[str, Any]) -> bool:
+        try:
+            health = self.retriever.update()
+        except Exception as exc:
+            self._fail(state, "bm25", f"keyword index build failed: {exc}")
+            return False
+        if not health.bm25_ready:
+            self._fail(state, "bm25", "keyword index did not become ready")
+            return False
+        self._record_progress(state)
+        self._mark(state, "bm25", "done", "keyword search current for the candidate revision")
+        return True
+
+    def _record_progress(self, state: dict[str, Any]) -> None:
+        completed = self.retriever._run(("status",), timeout=30)
+        counts = self.retriever._status_counts(completed.stdout if completed.returncode == 0 else "")
+        state["progress"] = {
+            "documents": counts["documents"],
+            "vectors": counts["vectors"],
+            "pending": counts["pending"],
+            "counted_at": _now(),
+        }
+        self.store.write(state)
+
+    def _run_embed(self, state: dict[str, Any], key: str = "embed") -> bool:
+        self._mark(state, key, "running", "embedding team memory")
+        # One `qmd embed` invocation proves nothing by itself: qmd embeds in
+        # bounded batches, a concurrent session's embedder can hold the embed
+        # lock (our pass prints "Skipping" and exits 0 while theirs runs), and
+        # freshly written vectors become countable slightly after the process
+        # exits. Drain instead: keep invoking embed and re-reading the real
+        # count for as long as pending falls; fail only after a full settle
+        # window passes with no progress from anyone.
+        try:
+            settle = float(
+                self.retriever.environment.get(
+                    "EGREGORE_UPGRADE_EMBED_SETTLE_SECONDS", "20"
+                )
+            )
+        except (AttributeError, TypeError, ValueError):
+            settle = 20.0
+        previous: int | None = None
+        deadline = time.monotonic() + settle
+        while True:
+            if self._cancelled(state):
+                return False
+            try:
+                self.retriever.embed_foreground()
+            except Exception as exc:
+                self._fail(state, key, f"meaning-index build failed: {exc}")
+                return False
+            self._record_progress(state)
+            pending = int(state["progress"].get("pending", 0) or 0)
+            if not pending:
+                break
+            if previous is None or pending < previous:
+                previous = pending
+                deadline = time.monotonic() + settle
+                vectors = state["progress"].get("vectors", 0)
+                self._mark(
+                    state, key, "running", f"{vectors} chunks embedded · {pending} pending"
+                )
+            elif time.monotonic() >= deadline:
+                self._fail(state, key, f"{pending} documents still awaiting embeddings after the build")
+                return False
+            if settle > 0:
+                time.sleep(min(2.0, settle))
+        self._mark(state, key, "done", f"{state['progress'].get('vectors', 0)} chunks embedded")
+        return True
+
+    def _run_delta(self, state: dict[str, Any]) -> bool:
+        # Canonical memory may have moved while the initial build ran. Converge:
+        # re-sync, re-index, embed the remaining delta — bounded, resumable.
+        for _ in range(5):
+            if self._cancelled(state):
+                return False
+            self.retriever._source_revision_cache = None
+            if self.retriever._semantic_state_current():
+                state["semantic_revision"] = self.retriever._semantic_source_revision()
+                state["source_revision"] = self.retriever._source_revision()
+                self._mark(state, "delta", "done", "meaning index matches the candidate revision")
+                return True
+            receipt = self.runtime.synchronize()
+            if not bool(getattr(receipt, "canonical_current", False)):
+                self._fail(state, "delta", "team memory changed and could not be re-synchronized")
+                return False
+            # Bring the keyword index to the new revision before embedding the
+            # delta — a no-op when synchronize already did it.
+            try:
+                self.retriever.update()
+            except Exception as exc:
+                self._fail(state, "delta", f"keyword index refresh failed: {exc}")
+                return False
+            if not self._run_embed(state, key="delta"):
+                return False
+            # _run_embed marked delta done; loop re-checks currency in case
+            # memory moved again during that embed.
+            self._mark(state, "delta", "running", "verifying the delta landed")
+        self._fail(state, "delta", "team memory kept changing during preparation; retry when quieter")
+        return False
+
+    def _run_verify(self, state: dict[str, Any]) -> bool:
+        from .readiness import build_report, session_boundary_state
+        from .contracts import Permission
+
+        health = self.retriever.health()
+        gates: dict[str, bool | str] = {}
+        gates["bm25_current"] = bool(health.bm25_ready and health.index_source_revision == health.source_revision)
+        gates["semantic_complete"] = bool(health.semantic_ready)
+        gates["semantic_matches_source"] = bool(
+            health.semantic_source_revision is not None
+            and health.semantic_source_revision == health.source_revision
+        )
+        gates["reranker_disabled"] = not RERANKING_ENABLED
+        boundary = session_boundary_state(self.repository_root)
+        authorization: Any
+        try:
+            actor = self.runtime.resolve_actor(
+                session_id=os.environ.get("EGREGORE_SESSION_ID", "upgrade-verify"),
+                harness="upgrade",
+            )
+            authorization = self.runtime.authorize(actor, Permission.READ)
+            gates["identity_resolves"] = True
+            gates["authorization_allows"] = bool(authorization.allowed)
+        except Exception as exc:
+            authorization = exc
+            gates["identity_resolves"] = False
+            gates["authorization_allows"] = False
+        report = build_report(
+            root=self.repository_root,
+            health=health,
+            boundary=boundary,
+            authorization=authorization if not isinstance(authorization, bool) else None,
+        )
+        stages = {item.key: item.stage for item in report.items}
+        gates["boundary_ready"] = stages.get("boundary") == "ready"
+        gates["hybrid_probe"] = stages.get("search") == "ready" and gates["semantic_complete"]
+        # Retrieval in the upgraded Runtime is QMD-only by construction: the
+        # engine never consults graph projections and records that contract.
+        gates["graph_retrieval"] = "not-used-by-this-runtime"
+        state["gates"] = gates
+        state["semantic_revision"] = health.semantic_source_revision
+        failed = [name for name, value in gates.items() if value is False]
+        if failed:
+            self._fail(state, "verify", "gates failed: " + ", ".join(failed))
+            return False
+        self._mark(state, "verify", "done", "all activation gates passed")
+        return True
+
+    # -- driver ----------------------------------------------------------
+
+    def prepare(self) -> dict[str, Any]:
+        state = self.store.load()
+        if state is None:
+            raise UpgradeError("no upgrade has been initialized for this instance")
+        if state.get("status") == STATUS_ACTIVE:
+            return state
+        self.store.clear_cancel()
+        state["status"] = STATUS_PREPARING
+        state["worker"] = {"pid": os.getpid(), "started_at": _now()}
+        self.store.write(state)
+
+        runners = {
+            "sync": self._run_sync,
+            "candidate": self._run_candidate,
+            "install": self._run_install,
+            "node": self._run_node,
+            "qmd": self._run_qmd,
+            "bm25": self._run_bm25,
+            "embed": self._run_embed,
+            "delta": self._run_delta,
+            "verify": self._run_verify,
+        }
+        for key, _label in STAGES:
+            if self._cancelled(state):
+                return state
+            if self._stage_done(state, key):
+                continue
+            self._mark(state, key, "running")
+            if not runners[key](state):
+                return state
+        state["status"] = STATUS_READY
+        self.store.write(state)
+        return state
+
+    def _apply_candidate_bytes(self, state: dict[str, Any]) -> dict[str, Any]:
+        """Copy the installed candidate bundle into the instance, backing up
+        every replaced file so rollback restores exact previous bytes."""
+        import shutil
+        import tarfile
+
+        package_root = Path(state.get("installed_path") or "")
+        if not (package_root / "runtime" / APPLY_TREES[0]).is_dir():
+            raise UpgradeError("candidate bundle tree is missing; re-run preparation")
+        instance = Path(state["instance_path"])
+        replaced: list[str] = []
+        added: list[str] = []
+        selected: dict[str, Path] = {}
+        for tree_name in APPLY_TREES:
+            tree = package_root / "runtime" / tree_name
+            if not tree.is_dir():
+                continue
+            for source_file in tree.rglob("*"):
+                if not source_file.is_file():
+                    continue
+                relative = str(source_file.relative_to(tree))
+                selected.setdefault(relative, source_file)
+        backup_path = self.store.root / "rollback-files.tar.gz"
+        with tarfile.open(backup_path, "w:gz") as backup:
+            for relative in sorted(selected):
+                destination = instance / relative
+                if destination.exists():
+                    backup.add(destination, arcname=relative)
+                    replaced.append(relative)
+                else:
+                    added.append(relative)
+        for relative in sorted(selected):
+            destination = instance / relative
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(selected[relative], destination)
+        manifest = {"replaced": replaced, "added": added, "trees": list(APPLY_TREES)}
+        (self.store.root / "rollback-manifest.json").write_text(
+            json.dumps(manifest, indent=2) + "\n", encoding="utf-8"
+        )
+        return {"applied_files": len(selected), "replaced": len(replaced), "added": len(added)}
+
+    def activate(self) -> dict[str, Any]:
+        state = self.store.load()
+        if state is None or state.get("status") != STATUS_READY:
+            raise UpgradeError(
+                "the candidate Runtime is not ready to activate; every gate must pass first"
+            )
+        gates = state.get("gates", {})
+        if any(value is False for value in gates.values()):
+            raise UpgradeError("activation refused: a recorded gate is failing")
+        # The recorded tarball digest must still match the bytes on disk —
+        # a swapped or corrupted candidate never activates.
+        source = Path(state.get("candidate_source") or "")
+        if not source.is_file() or self._sha256(source) != state.get("candidate_digest"):
+            raise UpgradeError("candidate package digest no longer matches; re-run preparation")
+        applied = self._apply_candidate_bytes(state)
+        record = {
+            "schema_version": SCHEMA_VERSION,
+            "active_version": state["candidate_version"],
+            "rollback_version": state["rollback_version"],
+            "source_revision": state["source_revision"],
+            "semantic_revision": state["semantic_revision"],
+            "candidate_digest": state["candidate_digest"],
+            "installed_path": state["installed_path"],
+            "applied": applied,
+            "activated_at": _now(),
+            "retrieval": "runtime-qmd",
+            "graph_retrieval": "not-used-by-this-runtime",
+        }
+        # Atomic: the activation record replaces in one rename; the state file
+        # then records active status. A crash between the two leaves a valid
+        # activation record and a resumable state — never partial retrieval.
+        self.store.write_activation(record)
+        state["status"] = STATUS_ACTIVE
+        state["active_version"] = state["candidate_version"]
+        self.store.write(state)
+        return state
+
+    def rollback(self) -> dict[str, Any]:
+        import tarfile
+
+        state = self.store.load()
+        if state is None:
+            raise UpgradeError("no upgrade state exists for this instance")
+        activation = self.store.load_activation() or {}
+        target = activation.get("rollback_version") or state.get("rollback_version")
+        if not target:
+            raise UpgradeError("no rollback version is recorded for this instance")
+        # Restore exact previous bytes: replaced files come back from the
+        # backup archive, files the candidate added are removed.
+        backup_path = self.store.root / "rollback-files.tar.gz"
+        manifest_path = self.store.root / "rollback-manifest.json"
+        if activation.get("applied") and backup_path.is_file() and manifest_path.is_file():
+            instance = Path(state["instance_path"])
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+            with tarfile.open(backup_path, "r:gz") as backup:
+                for member in backup.getmembers():
+                    member_path = (instance / member.name).resolve()
+                    if not str(member_path).startswith(str(instance.resolve())):
+                        raise UpgradeError(f"unsafe path in rollback archive: {member.name}")
+                backup.extractall(instance)
+            for relative in manifest.get("added", ()):  # candidate-only files
+                candidate_file = (instance / relative).resolve()
+                if str(candidate_file).startswith(str(instance.resolve())):
+                    candidate_file.unlink(missing_ok=True)
+        record = {
+            "schema_version": SCHEMA_VERSION,
+            "active_version": target,
+            "rollback_version": target,
+            "rolled_back_from": state.get("candidate_version"),
+            "rolled_back_at": _now(),
+            "retrieval": "previous-runtime",
+        }
+        self.store.write_activation(record)
+        state["status"] = STATUS_ROLLED_BACK
+        state["active_version"] = target
+        self.store.write(state)
+        return state
+
+    def cancel(self) -> dict[str, Any]:
+        state = self.store.load()
+        if state is None:
+            raise UpgradeError("no upgrade state exists for this instance")
+        self.store.request_cancel()
+        worker = state.get("worker") or {}
+        pid = worker.get("pid")
+        if state.get("status") == STATUS_PREPARING and not (pid and _pid_alive(pid)):
+            # No live worker will observe the flag — record the cancel now.
+            state["status"] = STATUS_CANCELLED
+            self.store.write(state)
+        return self.store.load() or state
+
+
+__all__ = [
+    "SCHEMA_VERSION",
+    "STAGES",
+    "STATUS_ACTIVE",
+    "STATUS_CANCELLED",
+    "STATUS_FAILED",
+    "STATUS_PENDING",
+    "STATUS_PREPARING",
+    "STATUS_READY",
+    "STATUS_ROLLED_BACK",
+    "UpgradeEngine",
+    "UpgradeError",
+    "UpgradeStore",
+    "describe",
+    "new_state",
+    "upgrade_root",
+]