@ouro.bot/cli 0.1.0-alpha.771 → 0.1.0-alpha.773

package/dist/heart/daemon/daemon-bootstrap-startup.js

@@ -1,14 +1,37 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.PUBLIC_DAEMON_STARTUP_FAILURE_REASON = void 0;
+exports.createProviderReadinessPreparationFailure = createProviderReadinessPreparationFailure;
 exports.failFastContainerCredentialBootstrapStartup = failFastContainerCredentialBootstrapStartup;
 exports.startDaemonAfterContainerCredentialBootstrap = startDaemonAfterContainerCredentialBootstrap;
 const runtime_1 = require("../../nerves/runtime");
 const daemon_tombstone_1 = require("./daemon-tombstone");
 const REDACTED_BOOTSTRAP_STARTUP_ERROR = "container credential bootstrap rejected; recoverable claim retained for reconciliation";
+const REDACTED_DAEMON_PREPARATION_ERROR = "provider runtime preparation failed before startup; run `ouro doctor` for diagnosis";
+exports.PUBLIC_DAEMON_STARTUP_FAILURE_REASON = "startupFailurePublic";
+class DaemonPreparationFailure extends Error {
+    name = "DaemonPreparationFailure";
+}
+function createProviderReadinessPreparationFailure(issues) {
+    const lines = ["Provider checks need attention"];
+    for (const issue of issues) {
+        lines.push(issue.summary);
+        lines.push(...issue.actions.map((action) => `  ${action.actor}: ${action.command}`));
+    }
+    return new DaemonPreparationFailure(lines.join("\n"));
+}
 function failFastContainerCredentialBootstrapStartup(input) {
-    const error = new Error(REDACTED_BOOTSTRAP_STARTUP_ERROR);
+    failFastDaemonStartup({
+        exit: input.exit,
+        errorMessage: REDACTED_BOOTSTRAP_STARTUP_ERROR,
+        eventMessage: "daemon entrypoint failed before server startup",
+    });
+}
+function failFastDaemonStartup(input) {
+    const errorMessage = input.errorMessage;
+    const error = new Error(errorMessage);
     try {
-        (0, daemon_tombstone_1.writeDaemonTombstone)("startupFailure", error);
+        (0, daemon_tombstone_1.writeDaemonTombstone)(exports.PUBLIC_DAEMON_STARTUP_FAILURE_REASON, error);
     }
     catch {
         // Exit remains mandatory even if best-effort tombstone reporting fails.
@@ -18,8 +41,8 @@ function failFastContainerCredentialBootstrapStartup(input) {
             level: "error",
             component: "daemon",
             event: "daemon.entry_error",
-            message: "daemon entrypoint failed before server startup",
-            meta: { error: REDACTED_BOOTSTRAP_STARTUP_ERROR },
+            message: input.eventMessage,
+            meta: { error: errorMessage },
         });
     }
     catch {
@@ -36,6 +59,19 @@ async function startDaemonAfterContainerCredentialBootstrap(input) {
         failFastContainerCredentialBootstrapStartup({ exit: input.exit });
         return false;
     }
+    try {
+        await input.prepareDaemon?.();
+    }
+    catch (error) {
+        input.markStartupFailure();
+        const controlledMessage = error instanceof DaemonPreparationFailure ? error.message : null;
+        failFastDaemonStartup({
+            exit: input.exit,
+            errorMessage: controlledMessage ?? REDACTED_DAEMON_PREPARATION_ERROR,
+            eventMessage: controlledMessage ?? REDACTED_DAEMON_PREPARATION_ERROR,
+        });
+        return false;
+    }
     await input.startDaemon();
     return true;
 }

package/dist/heart/daemon/daemon-entry.js

@@ -74,7 +74,6 @@ const runtime_credentials_1 = require("../runtime-credentials");
 const machine_identity_1 = require("../machine-identity");
 const container_credential_bootstrap_1 = require("./container-credential-bootstrap");
 const daemon_bootstrap_startup_1 = require("./daemon-bootstrap-startup");
-const auth_flow_1 = require("../auth/auth-flow");
 const sanctuary_health_runner_1 = require("../../senses/sanctuary-health-runner");
 const sanctuary_acceptance_marker_1 = require("./sanctuary-acceptance-marker");
 const sanctuary_scheduler_liveness_1 = require("./sanctuary-scheduler-liveness");
@@ -588,77 +587,40 @@ function writeStopCommandHealthState() {
         // Health writes are best-effort during shutdown.
     }
 }
-function providerPreloadTargets(agent) {
-    try {
-        const { config } = (0, auth_flow_1.readAgentConfigForAgent)(agent, (0, identity_1.getAgentBundlesRoot)());
-        return [...new Set([config.humanFacing.provider, config.agentFacing.provider])];
-    }
-    catch (error) {
-        (0, runtime_1.emitNervesEvent)({
-            level: "warn",
-            component: "daemon",
-            event: "daemon.provider_preload_skipped",
-            message: "skipping provider credential preload because agent config could not be read",
-            meta: {
-                agent,
-                error: error instanceof Error ? error.message : /* v8 ignore next -- defensive non-Error config-read failures @preserve */ String(error),
-            },
-        });
-        return [];
-    }
-}
-function providerPreloadMissingTargets(result, providers) {
-    return providers.filter((provider) => !result.pool.providers[provider]);
-}
-async function preloadProviderCredentialPools() {
-    await Promise.all(managedAgents.map(async (agent) => {
-        const providers = providerPreloadTargets(agent);
-        if (providers.length === 0)
-            return;
-        const result = await (0, provider_credentials_1.refreshProviderCredentialPool)(agent, { preserveCachedOnFailure: true, providers });
-        if (result.ok) {
-            const missingProviders = providerPreloadMissingTargets(result, providers);
-            if (missingProviders.length > 0) {
-                (0, runtime_1.emitNervesEvent)({
-                    level: "warn",
-                    component: "daemon",
-                    event: "daemon.provider_preload_unavailable",
-                    message: "provider credential preload returned an incomplete selected provider cache",
-                    meta: {
-                        agent,
-                        reason: "missing",
-                        error: `missing selected providers: ${missingProviders.join(", ")}`,
-                    },
-                });
-                return;
-            }
-            const records = Object.values(result.pool.providers).filter((record) => !!record);
-            (0, provider_credentials_1.cacheProviderCredentialRecords)(agent, records, new Date(result.pool.updatedAt));
-            return;
+async function prepareProviderRuntime() {
+    const bundlesRoot = (0, identity_1.getAgentBundlesRoot)();
+    const readiness = await Promise.all(managedAgents.map(async (agent) => {
+        try {
+            const result = await (0, agent_config_check_1.checkAgentConfigWithProviderHealth)(agent, bundlesRoot);
+            if (result.ok)
+                return null;
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.provider_readiness_unavailable",
+                message: "fresh provider readiness was unavailable before daemon startup",
+                meta: { agent },
+            });
+            return result.issue ?? {
+                summary: `${agent}: provider runtime unavailable`,
+                actions: [{ actor: "agent-runnable", command: "ouro doctor" }],
+            };
+        }
+        catch {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.provider_readiness_unavailable",
+                message: "fresh provider readiness check failed before daemon startup",
+                meta: { agent },
+            });
+            throw new Error("provider runtime preparation failed");
         }
-        (0, runtime_1.emitNervesEvent)({
-            level: "warn",
-            component: "daemon",
-            event: "daemon.provider_preload_unavailable",
-            message: "provider credential preload could not refresh selected provider cache",
-            meta: {
-                agent,
-                reason: result.reason,
-                error: result.error,
-            },
-        });
     }));
-}
-function startProviderCredentialPoolPreload() {
-    return preloadProviderCredentialPools().catch((error) => {
-        (0, runtime_1.emitNervesEvent)({
-            level: "error",
-            component: "daemon",
-            event: "daemon.provider_preload_error",
-            message: "provider credential preload failed after daemon startup",
-            meta: { error: error instanceof Error ? error.message : /* v8 ignore next -- defensive non-Error provider preload failures @preserve */ String(error) },
-        });
-    });
+    const failures = readiness.filter((entry) => entry !== null);
+    if (failures.length > 0) {
+        throw (0, daemon_bootstrap_startup_1.createProviderReadinessPreparationFailure)(failures);
+    }
 }
 function scheduleStartupSentinelAfterProviderPreload(agent, preload) {
     void preload.then(async () => {
@@ -693,6 +655,7 @@ function scheduleStartupSentinelAfterProviderPreload(agent, preload) {
 /* v8 ignore start -- habit wiring: lambdas delegate to processManager/fs; tested via HabitScheduler unit tests @preserve */
 void (0, daemon_bootstrap_startup_1.startDaemonAfterContainerCredentialBootstrap)({
     loadBootstrap: () => (0, container_credential_bootstrap_1.loadContainerCredentialBootstrap)(managedAgents),
+    prepareDaemon: prepareProviderRuntime,
     startDaemon: () => daemon.start(),
     markStartupFailure: () => { _tombstoneWritten = true; },
     exit: (code) => process.exit(code),
@@ -700,7 +663,7 @@ void (0, daemon_bootstrap_startup_1.startDaemonAfterContainerCredentialBootstrap
     if (!started)
         return;
     supercronicSupervisor?.start();
-    const providerPreload = startProviderCredentialPoolPreload();
+    const providerPreload = Promise.resolve();
     const bundlesRoot = (0, identity_1.getAgentBundlesRoot)();
     const ouroPath = supercronicSupervisor
         ? "/usr/local/bin/node /opt/ouro/dist/heart/daemon/ouro-entry.js"

package/dist/heart/daemon/daemon.js

@@ -618,6 +618,7 @@ class OuroDaemon {
     senseAutostartTimer = null;
     externalEventReconcileTimer = null;
     externalEventReconcileRunning = false;
+    constructedAtMs = Date.now();
     mailboxServerFactory;
     privateRuntimePolicyDeps;
     onStopCommandComplete;
@@ -1503,7 +1504,7 @@ class OuroDaemon {
         }
         return { type: "message", privateTurnDecision: decision, ...(externalEvent ? { externalEvent } : {}) };
     }
-    queueExternalEventForPrivateRuntime(record) {
+    queueExternalEventForPrivateRuntime(record, quiet = false) {
         const pendingDir = path.join(this.bundlesRoot, `${record.agent}.ouro`, "state", "pending", pending_1.PRIVATE_RUNTIME_PENDING.friendId, pending_1.PRIVATE_RUNTIME_PENDING.channel, pending_1.PRIVATE_RUNTIME_PENDING.key);
         const originKey = `${record.source}:${record.eventId}`;
         const parsedReceivedAt = Date.parse(record.receivedAt);
@@ -1523,18 +1524,20 @@ class OuroDaemon {
             mode: "relay",
             packetId: `external-event:${record.agent}:${record.source}:${record.eventId}:generation:${record.generation}:attempt:${record.attemptCount}`,
         });
-        (0, runtime_1.emitNervesEvent)({
-            component: "daemon",
-            event: "daemon.external_event_private_runtime_queued",
-            message: "queued external event for private-runtime attention",
-            meta: {
-                agent: record.agent,
-                source: record.source,
-                eventType: record.eventType,
-                eventId: record.eventId,
-                pendingDir,
-            },
-        });
+        if (!quiet) {
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.external_event_private_runtime_queued",
+                message: "queued external event for private-runtime attention",
+                meta: {
+                    agent: record.agent,
+                    source: record.source,
+                    eventType: record.eventType,
+                    eventId: record.eventId,
+                    pendingDir,
+                },
+            });
+        }
     }
     externalEventRootPath() {
         return this.externalEventRoot ?? (0, router_1.getExternalEventRoot)();
@@ -1575,8 +1578,19 @@ class OuroDaemon {
                 source: primary.source,
                 eventType: primary.eventType,
                 eventId: primary.eventId,
-            }, receipt.id, primary.generation, primary.attemptCount), () => { for (const record of claimed)
-                this.queueExternalEventForPrivateRuntime(record); }, lease);
+            }, receipt.id, primary.generation, primary.attemptCount), () => {
+                const batched = claimed.length > 1;
+                for (const record of claimed)
+                    this.queueExternalEventForPrivateRuntime(record, batched);
+                if (batched) {
+                    (0, runtime_1.emitNervesEvent)({
+                        component: "daemon",
+                        event: "daemon.external_event_private_runtime_batch_queued",
+                        message: "queued an external-event batch for private-runtime attention",
+                        meta: { agent: primary.agent, source: primary.source, eventCount: claimed.length },
+                    });
+                }
+            }, lease);
             failureClass = wake.denialCode === "managed_runtime_unavailable"
                 ? "managed_runtime_unavailable"
                 : wake.data?.decision?.denialCode === "provider_lane_unavailable"
@@ -1616,6 +1630,9 @@ class OuroDaemon {
             }
             const statuses = (0, router_1.listExternalEventStatus)(this.externalEventRootPath());
             const dueRecords = [];
+            const providerEvidenceByAgent = new Map();
+            const runtimeEvidenceByAgent = new Map();
+            const recoveredByClass = new Map();
             for (const status of statuses) {
                 if (status.corrupt)
                     continue;
@@ -1629,15 +1646,35 @@ class OuroDaemon {
                     let evidence = null;
                     try {
                         if (failure.class === "provider_lane_unavailable") {
-                            const binding = (0, provider_binding_resolver_1.resolveEffectiveProviderBinding)({
-                                agentName: record.agent,
-                                agentRoot: path.join(this.bundlesRoot, `${record.agent}.ouro`),
-                                lane: "inner",
-                            });
-                            if (binding.ok && binding.binding.credential.status === "present"
-                                && binding.binding.readiness.status === "ready" && binding.binding.readiness.checkedAt
-                                && Date.parse(binding.binding.readiness.checkedAt) > Date.parse(failure.failedAt)) {
-                                evidence = { class: failure.class, observedAt: binding.binding.readiness.checkedAt };
+                            if (!providerEvidenceByAgent.has(record.agent)) {
+                                providerEvidenceByAgent.set(record.agent, null);
+                                const binding = (0, provider_binding_resolver_1.resolveEffectiveProviderBinding)({
+                                    agentName: record.agent,
+                                    agentRoot: path.join(this.bundlesRoot, `${record.agent}.ouro`),
+                                    lane: "inner",
+                                });
+                                providerEvidenceByAgent.set(record.agent, binding.ok && binding.binding.credential.status === "present"
+                                    && binding.binding.readiness.status === "ready" && binding.binding.readiness.checkedAt
+                                    ? { observedAt: binding.binding.readiness.checkedAt }
+                                    : null);
+                            }
+                            const current = providerEvidenceByAgent.get(record.agent);
+                            const readinessIsCurrentBoot = current && Date.parse(current.observedAt) >= this.constructedAtMs;
+                            if (current && ((0, router_1.isExactLegacyProviderRecoveryFailure)(record)
+                                ? readinessIsCurrentBoot
+                                : Date.parse(current.observedAt) > Date.parse(failure.failedAt))) {
+                                evidence = { class: failure.class, observedAt: current.observedAt };
+                            }
+                        }
+                        else if (failure.class === "execution_lease_expired") {
+                            if (!runtimeEvidenceByAgent.has(record.agent)) {
+                                const snapshot = this.processManager.listAgentSnapshots().find((candidate) => candidate.name === record.agent && candidate.channel === "private-runtime"
+                                    && candidate.status === "running" && candidate.pid !== null && candidate.startedAt !== null);
+                                runtimeEvidenceByAgent.set(record.agent, snapshot?.startedAt ? { observedAt: snapshot.startedAt } : null);
+                            }
+                            const current = runtimeEvidenceByAgent.get(record.agent);
+                            if (current && Date.parse(current.observedAt) > Date.parse(failure.failedAt)) {
+                                evidence = { class: failure.class, observedAt: current.observedAt };
                             }
                         }
                         else if (this.hasManagedPrivateRuntime(record.agent) && Date.parse(now) > Date.parse(failure.failedAt)) {
@@ -1645,8 +1682,9 @@ class OuroDaemon {
                         }
                         if (!evidence)
                             continue;
-                        const revival = (0, router_1.reviveExternalEventAfterRecovery)(record.recordPath, { expectedVersion: record.version, expectedGeneration: record.generation, evidence, now: () => now });
+                        const revival = (0, router_1.reviveExternalEventAfterRecovery)(record.recordPath, { expectedVersion: record.version, expectedGeneration: record.generation, evidence, now: () => now, quiet: true });
                         record = revival.record;
+                        recoveredByClass.set(failure.class, (recoveredByClass.get(failure.class) ?? 0) + 1);
                     }
                     catch (error) {
                         (0, runtime_1.emitNervesEvent)({
@@ -1668,6 +1706,14 @@ class OuroDaemon {
                 if (due)
                     dueRecords.push(record);
             }
+            if (recoveredByClass.size > 0) {
+                (0, runtime_1.emitNervesEvent)({
+                    component: "daemon",
+                    event: "daemon.external_event_recovery_batch",
+                    message: "requeued external events after cached infrastructure recovery evidence",
+                    meta: { recovered: Object.fromEntries(recoveredByClass) },
+                });
+            }
             const batches = new Map();
             for (const record of dueRecords) {
                 const key = `${record.agent}\0${record.source}`;

package/dist/heart/daemon/sanctuary-acceptance-adapter.js

@@ -32,6 +32,9 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.proveAttemptedRecoveryWithoutRetry = void 0;
 exports.evaluateSanctuaryProviderReadinessContract = evaluateSanctuaryProviderReadinessContract;
@@ -45,6 +48,8 @@ exports.canonicalDockerIdFromUnraidPrefixedId = canonicalDockerIdFromUnraidPrefi
 exports.runSanctuaryProductionBoundaryProbe = runSanctuaryProductionBoundaryProbe;
 exports.createSanctuaryReadOnlyDenialScenarioDriver = createSanctuaryReadOnlyDenialScenarioDriver;
 exports.readDefaultSanctuaryScenarioFacts = readDefaultSanctuaryScenarioFacts;
+exports.callbackPlaybackSnapshot = callbackPlaybackSnapshot;
+exports.recordCallbackPlayback = recordCallbackPlayback;
 exports.executeSanctuaryAcceptanceAdapter = executeSanctuaryAcceptanceAdapter;
 exports.executeSanctuaryAcceptanceVaultProbe = executeSanctuaryAcceptanceVaultProbe;
 exports.executeSanctuaryAcceptanceRevokedProbe = executeSanctuaryAcceptanceRevokedProbe;
@@ -55,6 +60,7 @@ const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
 const node_net_1 = require("node:net");
 const path = __importStar(require("node:path"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const friends_1 = require("@ouro.bot/friends");
 const runtime_1 = require("../../nerves/runtime");
 const telegram_approval_runtime_1 = require("../../senses/telegram-approval-runtime");
@@ -99,6 +105,7 @@ const NETWORK_TIMEOUT_MS = 10_000;
 const KEY_DIRECTORY = "/boot/config/plugins/dynamix.my.servers/keys";
 const SELECTED_KEY_RECORD = "/run/ouro-acceptance/unraid-key.json";
 const TELEGRAM_OFFSET = "/home/ouro/AgentBundles/sanctuary.ouro/state/senses/telegram/offset.json";
+const CALLBACK_PLAYBACK_JOURNAL = "state/approvals/sanctuary-callback-playback.sqlite";
 const TELEGRAM_IDENTITY_KEY = "/home/ouro/AgentBundles/sanctuary.ouro/state/senses/telegram/identity.key";
 const TELEGRAM_AUDIT = `/home/ouro/AgentBundles/sanctuary.ouro/${telegram_audit_ledger_1.TELEGRAM_ACCEPTANCE_AUDIT_RELATIVE_PATH}`;
 const TELEGRAM_AUDIT_HEAD = `/home/ouro/AgentBundles/sanctuary.ouro/${telegram_audit_ledger_1.TELEGRAM_ACCEPTANCE_AUDIT_HEAD_RELATIVE_PATH}`;
@@ -244,6 +251,7 @@ function createSanctuaryAcceptanceAdapterDependencies(secretFd = 3, options = {}
         refreshMachine: runtime_credentials_1.refreshMachineRuntimeCredentialConfig,
         mergeMachine: runtime_credentials_1.mergeMachineRuntimeCredentialConfig,
         callbackProbe: executeSanctuaryAcceptanceCallbackProbe,
+        callbackPlaybackSnapshot: (coordinateDigest) => callbackPlaybackSnapshot((0, identity_1.getAgentRoot)(TARGET_ID), coordinateDigest),
         interactiveRuntime: executeSanctuaryInteractiveRuntimeOperation,
         hostRequest: options.hostRequest ?? ((payload) => defaultHostRequest(payload, hostBrokerSocket, adapterTimeoutMs)),
         telegramCredentials: () => (0, telegram_1.loadTelegramSenseCredentials)(TARGET_ID),
@@ -1912,6 +1920,94 @@ function callbackUpdate(value) {
     object(update.callback_query, "callback update callback_query");
     return update;
 }
+function withCallbackPlaybackJournal(agentRoot, operation) {
+    const databasePath = path.join(agentRoot, CALLBACK_PLAYBACK_JOURNAL);
+    (0, node_fs_1.mkdirSync)(path.dirname(databasePath), { recursive: true, mode: 0o700 });
+    const database = new better_sqlite3_1.default(databasePath);
+    try {
+        (0, node_fs_1.chmodSync)(databasePath, 0o600);
+        database.pragma("journal_mode = DELETE");
+        database.pragma("synchronous = FULL");
+        database.exec(`
+      CREATE TABLE IF NOT EXISTS callback_playback (
+        coordinate_digest TEXT PRIMARY KEY NOT NULL CHECK(length(coordinate_digest) = 64),
+        playback_count INTEGER NOT NULL CHECK(playback_count > 0)
+      )
+    `);
+        return operation(database);
+    }
+    finally {
+        database.close();
+    }
+}
+function callbackPlaybackSnapshot(agentRoot, coordinateDigest) {
+    if (!SHA256.test(coordinateDigest))
+        throw new Error("callback coordinate digest is invalid");
+    return withCallbackPlaybackJournal(agentRoot, (database) => {
+        const rows = database.prepare("SELECT coordinate_digest, playback_count FROM callback_playback ORDER BY coordinate_digest")
+            .all();
+        if (rows.some((row) => !SHA256.test(row.coordinate_digest) || !Number.isSafeInteger(row.playback_count) || row.playback_count <= 0)) {
+            throw new Error("callback playback journal is invalid");
+        }
+        const playbackCount = rows.find((row) => row.coordinate_digest === coordinateDigest)?.playback_count ?? 0;
+        return { playbackCount, coordinateDigest, journalDigest: sha256(JSON.stringify(rows)) };
+    });
+}
+function recordCallbackPlayback(agentRoot, coordinateDigest) {
+    if (!SHA256.test(coordinateDigest))
+        throw new Error("callback coordinate digest is invalid");
+    withCallbackPlaybackJournal(agentRoot, (database) => {
+        database.prepare(`
+      INSERT INTO callback_playback (coordinate_digest, playback_count) VALUES (?, 1)
+      ON CONFLICT(coordinate_digest) DO UPDATE SET playback_count = playback_count + 1
+    `).run(coordinateDigest);
+    });
+}
+function callbackCoordinate(update) {
+    const updateId = update.update_id;
+    if (!Number.isSafeInteger(updateId) || Number(updateId) < 0)
+        throw new Error("callback update_id is invalid");
+    const callback = object(update.callback_query, "callback update callback_query");
+    const from = object(callback.from, "callback update sender");
+    const message = object(callback.message, "callback update message");
+    const chat = object(message.chat, "callback update chat");
+    const callbackId = (value, label) => {
+        if (!Number.isSafeInteger(value) || Number(value) <= 0)
+            throw new Error(`${label} is invalid`);
+        return String(value);
+    };
+    const coordinate = {
+        updateId: Number(updateId),
+        queryId: text(callback.id, "callback query id"),
+        callbackData: text(callback.data, "callback data"),
+        userId: callbackId(from.id, "callback user id"),
+        chatId: callbackId(chat.id, "callback chat id"),
+        messageId: callbackId(message.message_id, "callback message id"),
+    };
+    return {
+        updateId: coordinate.updateId,
+        digest: sha256(`ouroboros.sanctuary.callback-coordinate.v1\0${JSON.stringify(coordinate)}`),
+    };
+}
+function callbackPlaybackPreflight(payload, deps) {
+    exactKeys(payload, ["operation", "update"], "callback playback preflight request");
+    const coordinate = callbackCoordinate(callbackUpdate(payload.update));
+    const offset = object(JSON.parse(fixedFile(deps, TELEGRAM_OFFSET)), "Telegram offset");
+    exactKeys(offset, ["nextUpdateId"], "Telegram offset");
+    if (!Number.isSafeInteger(offset.nextUpdateId) || Number(offset.nextUpdateId) < 0)
+        throw new Error("Telegram offset is invalid");
+    const snapshot = object(dependency(deps.callbackPlaybackSnapshot, "callback playback journal")(coordinate.digest), "callback playback journal snapshot");
+    exactKeys(snapshot, ["coordinateDigest", "journalDigest", "playbackCount"], "callback playback journal snapshot");
+    if (snapshot.coordinateDigest !== coordinate.digest || typeof snapshot.journalDigest !== "string" || !SHA256.test(snapshot.journalDigest)
+        || !Number.isSafeInteger(snapshot.playbackCount) || Number(snapshot.playbackCount) < 0) {
+        throw new Error("callback playback journal snapshot is invalid");
+    }
+    return {
+        playbackCount: Math.max(Number(snapshot.playbackCount), Number(offset.nextUpdateId) > coordinate.updateId ? 1 : 0),
+        coordinateDigest: coordinate.digest,
+        journalDigest: snapshot.journalDigest,
+    };
+}
 async function concurrentCallbackProbe(payload, deps) {
     const update = callbackUpdate(payload.update);
     if (!Number.isSafeInteger(payload.concurrency) || payload.concurrency < 2 || payload.concurrency > 16) {
@@ -2026,6 +2122,12 @@ async function storeKey(payload, deps) {
     if (text(handles[id], "vault-backed Unraid credential") !== text(stored[field], "vault-backed Unraid credential")) {
         throw new Error("Unraid key handle does not bind to the active vault field");
     }
+    const acknowledged = object(await dependency(deps.hostRequest, "Sanctuary host broker")({
+        operation: "acknowledge_key_storage", targetServerId: TARGET_SERVER_ID, keyId: id,
+    }), "Unraid key storage acknowledgement");
+    if (acknowledged.acknowledged !== true || acknowledged.id !== id) {
+        throw new Error("Unraid key storage acknowledgement failed");
+    }
     return { stored: true, keyId: id };
 }
 async function probeKey(payload, deps) {
@@ -2232,6 +2334,9 @@ async function executeSanctuaryAcceptanceAdapter(rawPayload, deps = createSanctu
             case "snapshot":
                 result = cursorSnapshot(payload, deps);
                 break;
+            case "callback_playback_preflight":
+                result = callbackPlaybackPreflight(payload, deps);
+                break;
             case "inject_callbacks_concurrently":
                 result = await concurrentCallbackProbe(payload, deps);
                 break;
@@ -2418,7 +2523,7 @@ async function executeSanctuaryInteractiveRuntimeOperation(rawPayload, supplied)
     };
     return (0, sanctuary_interactive_control_1.executeSanctuaryInteractiveEngine)(rawPayload, deps);
 }
-async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, _replay, deps = {
+async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, replay, deps = {
     refresh: runtime_credentials_1.refreshRuntimeCredentialConfig,
     credentials: telegram_1.loadTelegramSenseCredentials,
     identityKey: telegram_1.readOrCreateTelegramIdentityKey,
@@ -2426,6 +2531,7 @@ async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, _replay, deps
     createRuntime: telegram_approval_runtime_1.createTelegramApprovalRuntime,
     toolContext: sanctuary_runtime_1.createSanctuaryToolContext,
     effects: createAcceptanceProbeEffects,
+    recordCallbackPlayback: (coordinateDigest) => recordCallbackPlayback((0, identity_1.getAgentRoot)(TARGET_ID), coordinateDigest),
 }) {
     const update = callbackUpdate(rawUpdate);
     const refreshed = await deps.refresh(TARGET_ID);
@@ -2453,7 +2559,11 @@ async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, _replay, deps
         effects: effectBoundary?.port ?? unavailableEffects,
     });
     try {
+        deps.recordCallbackPlayback(callbackCoordinate(update).digest);
         const result = await runtime.transport.handleUpdate(update);
+        if (replay && (result.handled !== true || result.accepted !== false || result.reason !== "stale_callback")) {
+            throw new Error("Telegram callback replay did not settle as stale");
+        }
         return {
             settled: result.handled,
             claimed: result.reason === "accepted" || result.reason === "decision_refused",