@axiom-lattice/gateway 4.7.0 → 4.8.0

package/dist/index.mjs

@@ -9,13 +9,14 @@ import {
 } from "./chunk-73QZQQOT.mjs";
 import {
   buildPreviewTokenScript
-} from "./chunk-2POW27EI.mjs";
-import "./chunk-WKNZ7GBC.mjs";
+} from "./chunk-7CG57LEF.mjs";
+import "./chunk-EXPERMZR.mjs";
 import {
   CollectionService
 } from "./chunk-PWHUWFXL.mjs";
 import {
   createOrphanRecoveryRuntime,
+  createTaskRunLiveness,
   isProjectLifecycleTask,
   isRosterExemptProject,
   recoverTask,
@@ -23,7 +24,7 @@ import {
   resolveTaskOwnerMembership,
   resolveTopCaller,
   runOrphanRecovery
-} from "./chunk-KDQKTU3N.mjs";
+} from "./chunk-ZBTG3T7K.mjs";
 import {
   createRecoveryCallerDeps
 } from "./chunk-NOXGKYUV.mjs";
@@ -858,9 +859,11 @@ async function getAssistantStats(request, reply) {
 // src/controllers/run.ts
 import { v4 } from "uuid";
 import {
-  agentInstanceManager as agentInstanceManager2
+  agentInstanceManager as agentInstanceManager2,
+  getReplicaId,
+  getThreadRunOwner
 } from "@axiom-lattice/core";
-import { MessageChunkTypes } from "@axiom-lattice/protocols";
+import { MessageChunkTypes, AGENT_AFFINITY_HEADER, AFFINITY_ECHO_HEADER, affinityEchoHeaders, REPLICA_HEADER, AFFINITY_MISS_CODE, buildAffinityMissChunk } from "@axiom-lattice/protocols";
 
 // src/utils/agentScope.ts
 function headerValue(value) {
@@ -960,6 +963,24 @@ async function writeSse(raw, value, closed) {
 }
 
 // src/controllers/run.ts
+var replicaHeader = () => ({ [REPLICA_HEADER]: getReplicaId() });
+var exposeHeaders = `${REPLICA_HEADER}, ${AFFINITY_ECHO_HEADER}`;
+function applyAffinityEcho(request, reply) {
+  for (const [name, value] of Object.entries(affinityEchoHeaders(request.headers[AGENT_AFFINITY_HEADER]))) {
+    reply.header(name, value);
+  }
+}
+var AFFINITY_GRACE_MS = Number.isFinite(Number(process.env.AXIOM_AFFINITY_GRACE_MS)) ? Number(process.env.AXIOM_AFFINITY_GRACE_MS) : 500;
+async function peerOwnsThread(input) {
+  if (!process.env.AXIOM_REPLICA_ID) return null;
+  if (!input.tenant_id || !input.assistant_id || !input.thread_id) return null;
+  const owner = await getThreadRunOwner({
+    tenantId: input.tenant_id,
+    assistantId: input.assistant_id,
+    threadId: input.thread_id
+  });
+  return owner && owner !== getReplicaId() ? owner : null;
+}
 function getUserId2(request) {
   const authUser = request.user;
   if (authUser?.id) return authUser.id;
@@ -986,6 +1007,7 @@ var createRun = async (request, reply) => {
       ...user_id && { user_id }
     };
     if (!assistant_id) {
+      applyAffinityEcho(request, reply);
       reply.status(400).send({
         success: false,
         error: "\u52A9\u624BID\u662F\u5FC5\u9700\u7684"
@@ -1007,6 +1029,8 @@ var createRun = async (request, reply) => {
         command,
         custom_run_config: mergedConfig
       }, mode);
+      reply.header(REPLICA_HEADER, getReplicaId());
+      applyAffinityEcho(request, reply);
       reply.status(202).send({
         success: true,
         messageId: result.messageId,
@@ -1020,7 +1044,10 @@ var createRun = async (request, reply) => {
         "Content-Type": "text/event-stream",
         "Cache-Control": "no-cache",
         Connection: "keep-alive",
-        "Access-Control-Allow-Origin": "*"
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Expose-Headers": exposeHeaders,
+        ...replicaHeader(),
+        ...affinityEchoHeaders(request.headers[AGENT_AFFINITY_HEADER])
       });
       try {
         const messageInput = message_id ? { ...input, id: message_id } : input;
@@ -1031,6 +1058,18 @@ var createRun = async (request, reply) => {
         }, [MessageChunkTypes.MESSAGE_COMPLETED, MessageChunkTypes.INTERRUPT, MessageChunkTypes.MESSAGE_FAILED], {
           queueMode: mode
         });
+        let peer = await peerOwnsThread({ tenant_id, assistant_id, thread_id });
+        if (peer) {
+          await new Promise((resolve3) => setTimeout(resolve3, AFFINITY_GRACE_MS));
+          peer = await peerOwnsThread({ tenant_id, assistant_id, thread_id });
+        }
+        if (peer) {
+          reply.raw.write(`data: ${JSON.stringify(buildAffinityMissChunk(peer, thread_id))}
+
+`);
+          reply.raw.end();
+          return;
+        }
         await pipeAsyncIterableToSse(reply.raw, result.stream, {
           onError: (error) => [{
             type: "error",
@@ -1062,12 +1101,15 @@ var createRun = async (request, reply) => {
         command,
         custom_run_config: mergedConfig
       });
+      reply.header(REPLICA_HEADER, getReplicaId());
+      applyAffinityEcho(request, reply);
       reply.status(200).send({
         success: true,
         ...result
       });
     }
   } catch (error) {
+    applyAffinityEcho(request, reply);
     reply.status(500).send({
       success: false,
       error: `\u521B\u5EFA\u8FD0\u884C\u65F6\u53D1\u751F\u9519\u8BEF: ${error.message}`
@@ -1090,9 +1132,20 @@ var resumeStream = async (request, reply) => {
       "Content-Type": "text/event-stream",
       "Cache-Control": "no-cache",
       Connection: "keep-alive",
-      "Access-Control-Allow-Origin": "*"
+      "Access-Control-Allow-Origin": "*",
+      "Access-Control-Expose-Headers": exposeHeaders,
+      ...replicaHeader(),
+      ...affinityEchoHeaders(request.headers[AGENT_AFFINITY_HEADER])
     });
     try {
+      const peer = await peerOwnsThread({ tenant_id, assistant_id, thread_id });
+      if (peer) {
+        reply.raw.write(`data: ${JSON.stringify(buildAffinityMissChunk(peer, thread_id))}
+
+`);
+        reply.raw.end();
+        return;
+      }
       const agent = agentInstanceManager2.getAgent({
         assistant_id,
         thread_id,
@@ -1143,12 +1196,22 @@ var abortRun = async (request, reply) => {
       workspace_id,
       project_id
     });
+    const peer = await peerOwnsThread({ tenant_id, assistant_id: assistantId, thread_id: threadId });
+    if (peer) {
+      reply.header(REPLICA_HEADER, getReplicaId());
+      applyAffinityEcho(request, reply);
+      reply.status(409).send({ success: false, code: AFFINITY_MISS_CODE, owner: peer, threadId });
+      return;
+    }
     await agent.abort();
+    reply.header(REPLICA_HEADER, getReplicaId());
+    applyAffinityEcho(request, reply);
     reply.status(200).send({
       success: true,
       message: "Agent aborted"
     });
   } catch (error) {
+    applyAffinityEcho(request, reply);
     reply.status(500).send({
       success: false,
       error: `Abort failed: ${error.message}`
@@ -1166,13 +1229,17 @@ var recoverRun = async (request, reply) => {
       workspace_id,
       project_id
     });
-    const { status } = await agent.resumeTask();
+    const { status, leaseContended } = await agent.resumeTask();
+    reply.header(REPLICA_HEADER, getReplicaId());
+    applyAffinityEcho(request, reply);
     reply.status(200).send({
       success: true,
       threadId,
-      status
+      status,
+      ...leaseContended ? { code: AFFINITY_MISS_CODE } : {}
     });
   } catch (error) {
+    applyAffinityEcho(request, reply);
     reply.status(500).send({
       success: false,
       error: `Recover failed: ${error.message}`
@@ -7750,11 +7817,49 @@ function connectionRoutes(fastify2) {
 // src/controllers/mcp-configs.ts
 import {
   getStoreLattice as getStoreLattice9,
+  getReplicaId as getReplicaId2,
   mcpManager,
   McpLatticeManager,
   convertToConnection
 } from "@axiom-lattice/core";
 import { randomUUID as randomUUID10 } from "crypto";
+
+// src/services/mcpConfigSync.ts
+var MCP_CONFIG_CHANGED_TOPIC = "mcp-config-changed";
+async function publishMcpConfigChanged(outbox, owner, tenantId, serverKey) {
+  if (!tenantId || !serverKey) return;
+  try {
+    await outbox.append({
+      topic: MCP_CONFIG_CHANGED_TOPIC,
+      payload: { tenantId, serverKey },
+      originReplicaId: owner
+    });
+  } catch {
+  }
+}
+async function reconcileMcpConfigChange(payload, deps) {
+  const tenantId = payload?.tenantId;
+  const serverKey = payload?.serverKey;
+  if (typeof tenantId !== "string" || tenantId.length === 0) return;
+  if (typeof serverKey !== "string" || serverKey.length === 0) return;
+  const config = await deps.store.getConfigByKey(tenantId, serverKey);
+  if (!config || config.status !== "connected") {
+    if (deps.manager.hasServer(tenantId, serverKey)) {
+      await deps.manager.removeServer(tenantId, serverKey);
+    }
+    return;
+  }
+  const connection = deps.convertToConnection(config.config);
+  deps.manager.registerServer(tenantId, serverKey, connection, {
+    timeoutMs: config.config.timeout,
+    maxAttempts: config.config.retryAttempts,
+    selectedTools: config.selectedTools
+  });
+  await deps.manager.ensureConnected(tenantId, serverKey);
+  await deps.manager.registerToolsToToolLattice(tenantId, serverKey, config.selectedTools);
+}
+
+// src/controllers/mcp-configs.ts
 function getTenantId10(request) {
   const userTenantId = request.user?.tenantId;
   if (userTenantId) {
@@ -7762,6 +7867,14 @@ function getTenantId10(request) {
   }
   return request.headers["x-tenant-id"] || "default";
 }
+function broadcastMcpConfigChanged(tenantId, serverKey) {
+  if (!tenantId || !serverKey) return;
+  try {
+    const outbox = getStoreLattice9("default", "outbox").store;
+    void publishMcpConfigChanged(outbox, getReplicaId2(), tenantId, serverKey);
+  } catch {
+  }
+}
 async function getMcpServerConfigList(request, reply) {
   const tenantId = getTenantId10(request);
   try {
@@ -7839,6 +7952,7 @@ async function createMcpServerConfig(request, reply) {
       await store.updateConfig(tenantId, id3, { status: "error" });
       config.status = "error";
     }
+    broadcastMcpConfigChanged(tenantId, config.key);
     reply.code(201);
     return {
       success: true,
@@ -7868,7 +7982,9 @@ async function updateMcpServerConfig(request, reply) {
         message: "MCP server configuration not found"
       };
     }
-    const shouldReconnect = updates.config !== void 0;
+    const configChanged = updates.config !== void 0;
+    const toolsChanged = updates.selectedTools !== void 0;
+    const statusChanged = updates.status !== void 0;
     const updated = await store.updateConfig(tenantId, existing.id, updates);
     if (!updated) {
       return {
@@ -7876,7 +7992,7 @@ async function updateMcpServerConfig(request, reply) {
         message: "Failed to update MCP server configuration"
       };
     }
-    if (shouldReconnect) {
+    if (configChanged) {
       try {
         const connection = convertToConnection(updated.config);
         mcpManager.registerServer(tenantId, key, connection, {
@@ -7892,6 +8008,22 @@ async function updateMcpServerConfig(request, reply) {
         await store.updateConfig(tenantId, existing.id, { status: "error" });
         updated.status = "error";
       }
+    } else if (toolsChanged && mcpManager.hasServer(tenantId, key)) {
+      try {
+        await mcpManager.registerToolsToToolLattice(tenantId, key, updated.selectedTools);
+      } catch (error) {
+        console.warn("Failed to re-register MCP tools:", error);
+      }
+    }
+    if (statusChanged && updated.status !== "connected" && mcpManager.hasServer(tenantId, key)) {
+      try {
+        await mcpManager.removeServer(tenantId, key);
+      } catch (error) {
+        console.warn("Failed to remove MCP server:", error);
+      }
+    }
+    if (configChanged || toolsChanged || statusChanged) {
+      broadcastMcpConfigChanged(tenantId, key);
     }
     return {
       success: true,
@@ -7941,6 +8073,7 @@ async function deleteMcpServerConfig(request, reply) {
         message: "Failed to delete MCP server configuration"
       };
     }
+    broadcastMcpConfigChanged(tenantId, configKey);
     return {
       success: true,
       message: "MCP server configuration deleted successfully"
@@ -8015,12 +8148,17 @@ async function listMcpServerTools(request, reply) {
         message: "MCP server configuration not found"
       };
     }
-    if (!mcpManager.isServerConnected(tenantId, key)) {
+    const wasConnected = mcpManager.isServerConnected(tenantId, key);
+    if (!wasConnected) {
       await connectAndRegisterTools(config);
     }
-    if (config.status !== "connected") {
+    const statusStale = config.status !== "connected";
+    if (statusStale) {
       await store.updateConfig(tenantId, config.id, { status: "connected" });
     }
+    if (!wasConnected || statusStale) {
+      broadcastMcpConfigChanged(tenantId, key);
+    }
     const tools = await mcpManager.getServerTools(tenantId, key);
     return {
       success: true,
@@ -8055,6 +8193,7 @@ async function connectMcpServer(request, reply) {
     const updated = await store.updateConfig(tenantId, config.id, {
       status: "connected"
     });
+    broadcastMcpConfigChanged(tenantId, key);
     return {
       success: true,
       message: "MCP server connected successfully",
@@ -8068,6 +8207,7 @@ async function connectMcpServer(request, reply) {
     if (config) {
       await store.updateConfig(tenantId, config.id, { status: "error" });
     }
+    broadcastMcpConfigChanged(tenantId, key);
     return {
       success: false,
       message: `Failed to connect MCP server: ${error instanceof Error ? error.message : "Unknown error"}`
@@ -8094,6 +8234,7 @@ async function disconnectMcpServer(request, reply) {
     const updated = await store.updateConfig(tenantId, config.id, {
       status: "disconnected"
     });
+    broadcastMcpConfigChanged(tenantId, key);
     return {
       success: true,
       message: "MCP server disconnected successfully",
@@ -10619,8 +10760,8 @@ function snapshotPlainObject(value) {
   }
 }
 async function getInstallationStore() {
-  const { getStoreLattice: getStoreLattice29 } = await import("@axiom-lattice/core");
-  const store = getStoreLattice29("default", "channelInstallation").store;
+  const { getStoreLattice: getStoreLattice30 } = await import("@axiom-lattice/core");
+  const store = getStoreLattice30("default", "channelInstallation").store;
   if (store) return store;
   const { PostgreSQLChannelInstallationStore } = await import("@axiom-lattice/pg-stores");
   const databaseUrl = process.env.DATABASE_URL;
@@ -12369,6 +12510,10 @@ var ProjectTaskExecutionService = class {
         input.inputMessageId,
         true
       );
+      if (resumed.leaseContended) {
+        settlement.cleanup();
+        return { kind: "deferred", task: input.task, thread: input.thread, messageId: input.inputMessageId };
+      }
       if (resumed.status === ThreadStatus3.INTERRUPTED && settlement.tryClaim()) {
         settlement.cleanup();
         const outcome = await this.settleInterruptedEvent(input.task, input.thread, input.inputMessageId);
@@ -14056,7 +14201,8 @@ import { z as z5, ZodError as ZodError2 } from "zod";
 
 // src/services/agent-web-app/AgentWebAppRuntimeService.ts
 import { createHash as createHash4, randomUUID as randomUUID14 } from "crypto";
-import { MessageChunkTypes as MessageChunkTypes3, parseAgentWebAppGenUIBlock } from "@axiom-lattice/protocols";
+import { MessageChunkTypes as MessageChunkTypes3, parseAgentWebAppGenUIBlock, buildAffinityMissChunk as buildAffinityMissChunk2 } from "@axiom-lattice/protocols";
+import { getReplicaId as getReplicaId3, getThreadRunOwner as getThreadRunOwner2 } from "@axiom-lattice/core";
 function isRecord2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -14218,6 +14364,20 @@ var AgentWebAppRuntimeService = class {
   async resumeStream(webAppId, userId, threadId, messageId, selection) {
     const runtime = await this.resolveRuntime(webAppId, selection);
     await this.requireOwnedThread(runtime, userId, threadId);
+    if (process.env.AXIOM_REPLICA_ID) {
+      const owner = await getThreadRunOwner2({
+        tenantId: runtime.webApp.tenantId,
+        assistantId: runtime.webApp.assistantId,
+        threadId
+      });
+      if (owner && owner !== getReplicaId3()) {
+        return {
+          stream: (async function* () {
+            yield buildAffinityMissChunk2(owner, threadId);
+          })()
+        };
+      }
+    }
     const agent = this.resolveAgent(runtime, threadId);
     return { stream: agent.chunkStream(messageId, []) };
   }
@@ -14562,7 +14722,8 @@ async function verifyWebAppToken(token, issuer, options = {}) {
 }
 
 // src/routes/agent-web-app-runtime.ts
-import { createAgentWebAppStreamProjectionContext, projectAgentWebAppChunk } from "@axiom-lattice/protocols";
+import { createAgentWebAppStreamProjectionContext, projectAgentWebAppChunk, AGENT_AFFINITY_HEADER as AGENT_AFFINITY_HEADER2, AFFINITY_ECHO_HEADER as AFFINITY_ECHO_HEADER2, affinityEchoHeaders as affinityEchoHeaders2, REPLICA_HEADER as REPLICA_HEADER2 } from "@axiom-lattice/protocols";
+import { getReplicaId as getReplicaId4 } from "@axiom-lattice/core";
 var idSchema = z5.string().min(1).max(128);
 var paramsSchema = z5.object({ webAppId: idSchema }).strict();
 var threadParamsSchema = z5.object({ webAppId: idSchema, threadId: idSchema }).strict();
@@ -14711,7 +14872,7 @@ function registerAgentWebAppRuntimeRoutes(app2, deps) {
         const identity = await resolveRuntimeIdentity(request, webAppId, identities);
         const input = streamBodySchema.parse(request.body);
         const result = await deps.service.streamMessage(webAppId, identity.userId, threadId, withPreview(input, identity.preview));
-        beginSse(reply);
+        beginSse(reply, request);
         await pipeNativeWebAppStream(reply.raw, result.stream);
       } catch (error) {
         return normalizeError2(reply, error);
@@ -14723,7 +14884,7 @@ function registerAgentWebAppRuntimeRoutes(app2, deps) {
         const identity = await resolveRuntimeIdentity(request, webAppId, identities);
         const input = streamBodySchema.parse(request.body);
         const result = await deps.service.streamMessage(webAppId, identity.userId, threadId, withPreview(input, identity.preview));
-        beginSse(reply);
+        beginSse(reply, request);
         await pipeAgentWebAppStream(reply.raw, result.stream, result);
       } catch (error) {
         return normalizeError2(reply, error);
@@ -14735,7 +14896,7 @@ function registerAgentWebAppRuntimeRoutes(app2, deps) {
         const identity = await resolveRuntimeIdentity(request, webAppId, identities);
         const { messageId, projectId, modelKey: modelKey2 } = resumeSchema.parse(request.body);
         const result = await deps.service.resumeStream(webAppId, identity.userId, threadId, messageId, withPreview({ projectId, modelKey: modelKey2 }, identity.preview));
-        beginSse(reply);
+        beginSse(reply, request);
         await pipeNativeWebAppStream(reply.raw, result.stream);
       } catch (error) {
         return normalizeError2(reply, error);
@@ -14768,7 +14929,7 @@ function registerAgentWebAppRuntimeRoutes(app2, deps) {
         const identity = await resolveRuntimeIdentity(request, webAppId, identities);
         const { projectId, modelKey: modelKey2, message, response } = resumeBodySchema.parse(request.body);
         const result = await deps.service.resumeInterrupt(webAppId, identity.userId, threadId, interruptId, withPreview({ projectId, modelKey: modelKey2 }, identity.preview), { message, response });
-        beginSse(reply);
+        beginSse(reply, request);
         await pipeAgentWebAppStream(reply.raw, result.stream, result);
       } catch (error) {
         return normalizeError2(reply, error);
@@ -14996,6 +15157,10 @@ function registerAgentWebAppRuntimeRoutes(app2, deps) {
           if (!threadId) throw new AgentWebAppRuntimeError("INVALID_REQUEST", 400, "thread_id is required");
           const content = body.messages.map((message) => message.content).join("\n");
           const { messageId } = await deps.service.enqueueMessage(webAppId, identity.userId, threadId, content, { projectId: body.projectId, modelKey: body.modelKey });
+          reply.header(REPLICA_HEADER2, getReplicaId4());
+          for (const [name, value] of Object.entries(affinityEchoHeaders2(request.headers[AGENT_AFFINITY_HEADER2]))) {
+            reply.header(name, value);
+          }
           return reply.code(202).send({ success: true, messageId, queued: true });
         } catch (error) {
           return normalizeError2(reply, error);
@@ -15016,7 +15181,7 @@ function registerAgentWebAppRuntimeRoutes(app2, deps) {
           const identity = await resolveRuntimeIdentity(request, webAppId, identities);
           const body = resumeStreamSchema2.parse(request.body ?? {});
           const result = await deps.service.resumeStream(webAppId, identity.userId, body.thread_id, body.message_id, { projectId: body.projectId, modelKey: body.modelKey });
-          beginSse(reply);
+          beginSse(reply, request);
           await pipeNativeWebAppStream(reply.raw, result.stream);
         } catch (error) {
           return normalizeError2(reply, error);
@@ -15069,14 +15234,17 @@ async function pipeNativeWebAppStream(raw, stream) {
     }]
   });
 }
-function beginSse(reply) {
+function beginSse(reply, request) {
   reply.hijack();
   reply.raw.writeHead(200, {
     "Content-Type": "text/event-stream; charset=utf-8",
     "Cache-Control": "no-cache, no-transform",
     Connection: "keep-alive",
     "X-Accel-Buffering": "no",
-    "Access-Control-Allow-Origin": "*"
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Expose-Headers": `${REPLICA_HEADER2}, ${AFFINITY_ECHO_HEADER2}`,
+    [REPLICA_HEADER2]: getReplicaId4(),
+    ...affinityEchoHeaders2(request.headers[AGENT_AFFINITY_HEADER2])
   });
 }
 async function handle2(reply, operation) {
@@ -15676,6 +15844,17 @@ var ModelProviderServiceError = class extends Error {
     this.name = "ModelProviderServiceError";
   }
 };
+var UPSTREAM_ERROR_CODES = {
+  UPSTREAM_UNREACHABLE: true,
+  UPSTREAM_AUTH_FAILED: true,
+  UPSTREAM_REDIRECT_BLOCKED: true,
+  UPSTREAM_INVALID_RESPONSE: true,
+  UPSTREAM_RATE_LIMITED: true,
+  UPSTREAM_TIMEOUT: true
+};
+function isUpstreamErrorCode(code) {
+  return Object.prototype.hasOwnProperty.call(UPSTREAM_ERROR_CODES, code);
+}
 
 // src/services/model-provider/discovery.ts
 var MAX_DISCOVERED_MODELS = 1e3;
@@ -15951,6 +16130,33 @@ var ModelProviderService = class {
       models
     };
   }
+  /**
+   * Hydrates the runtime model registry from the persisted provider catalog.
+   *
+   * @returns The model keys registered in this process.
+   * @remarks Registers the Phase-1 (default tenant) enabled provider models so a
+   * replica that did not perform discovery — or that restarted — can execute
+   * them. Best-effort; the caller logs failures.
+   */
+  async hydrateRegistry() {
+    const tenantId = PHASE1_REGISTRY_TENANT;
+    if (!this.shouldRegister(tenantId)) {
+      return [];
+    }
+    const providers = await this.deps.providerStore.listProviders(tenantId);
+    const registered = [];
+    for (const provider of providers) {
+      if (!provider.enabled) {
+        continue;
+      }
+      const models = await this.deps.providerStore.listModels(
+        tenantId,
+        provider.id
+      );
+      registered.push(...this.syncRegistry(tenantId, provider, models));
+    }
+    return registered;
+  }
   /**
    * Create a provider configuration. Does not run discovery.
    */
@@ -16190,7 +16396,7 @@ var ModelProviderService = class {
         models
       };
     } catch (error) {
-      if (error instanceof ModelProviderServiceError) {
+      if (error instanceof ModelProviderServiceError && isUpstreamErrorCode(error.code)) {
         return {
           reachable: false,
           latencyMs: Date.now() - startedAt,
@@ -16606,6 +16812,10 @@ var ModelProviderController = class {
       return sendError2(reply, error);
     }
   }
+  /** Hydrates the runtime model registry from persisted providers (startup). */
+  async hydrateRegistry() {
+    return this.service.hydrateRegistry();
+  }
   /** GET /api/model-providers */
   async listProviders(request, reply) {
     try {
@@ -19174,6 +19384,8 @@ function createRoomMessageRetentionJob(input) {
 
 // src/services/collaboration/ProjectTaskExecutionMutex.ts
 import { snapshotExactRecord as snapshotExactRecord8 } from "@axiom-lattice/protocols";
+import { getCoordinationLeaseStore, getReplicaId as getReplicaId5, withLease } from "@axiom-lattice/core";
+var MACHINE_LEASE_OPTIONS = { ttlMs: 6e4, acquireTimeoutMs: 9e4, retryDelayMs: 100 };
 function machineKey(scope) {
   const snapshot = snapshotExactRecord8(scope, ["tenantId", "assistantId", "projectId"]);
   if (!snapshot) throw new Error("Invalid Project task machine scope");
@@ -19197,9 +19409,12 @@ var ProjectTaskExecutionMutex = class {
    */
   async runExclusive(scope, operation, signal) {
     const key = machineKey(scope);
+    const leaseKey = `project-task-machine:${encodeURIComponent(scope.tenantId)}:${encodeURIComponent(scope.assistantId)}:${encodeURIComponent(scope.projectId)}`;
     await this.acquire(key, signal);
     try {
-      return await operation();
+      const store = getCoordinationLeaseStore();
+      if (!store) return await operation();
+      return await withLease(store, leaseKey, getReplicaId5(), operation, MACHINE_LEASE_OPTIONS);
     } finally {
       this.release(key);
     }
@@ -19258,6 +19473,20 @@ import {
 import {
   snapshotExactRecord as snapshotExactRecord9
 } from "@axiom-lattice/protocols";
+
+// src/services/collaboration/projectTaskDispatchMarker.ts
+var EXECUTION_REQUESTED_AT_KEY = "execution_requested_at";
+function hasExecutionRequest(task) {
+  const value = task.context?.[EXECUTION_REQUESTED_AT_KEY];
+  if (typeof value !== "string" || value.length === 0) return false;
+  const parsed = new Date(value);
+  return Number.isFinite(parsed.getTime()) && parsed.toISOString() === value;
+}
+function withExecutionRequest(context, nowIso) {
+  return { ...context ?? {}, [EXECUTION_REQUESTED_AT_KEY]: nowIso };
+}
+
+// src/services/collaboration/ProjectTaskObserver.ts
 var SYSTEM_ACTOR2 = "system";
 var TERMINAL_ACTIONS = ["completed", "failed", "interrupted", "cancelled"];
 var ProjectTaskObserver = class {
@@ -19461,7 +19690,7 @@ var ProjectTaskObserver = class {
           return;
         }
       }
-      if (intent === "dispatch" && await this.isReady(task) && !signal?.aborted && !this.disposed) {
+      if ((intent === "dispatch" || hasExecutionRequest(task)) && await this.isReady(task) && !signal?.aborted && !this.disposed) {
         await this.runStart(task, signal);
       }
       return;
@@ -19578,7 +19807,7 @@ var ProjectTaskObserver = class {
     }, signal);
   }
   async mapOutcome(outcome) {
-    if (outcome.kind === "aborted") return;
+    if (outcome.kind === "aborted" || outcome.kind === "deferred") return;
     const current = snapshotTaskItem(await this.deps.tasks.getById(outcome.task.tenantId, outcome.task.id));
     if (!current || current.ownerId !== outcome.thread.assistantId || current.context?.thread_id !== outcome.thread.id) return;
     if (isTerminal(current.status) || current.status === "review") return;
@@ -20510,6 +20739,7 @@ var DEFAULT_MAX_EVENT_AGE_MS = 10 * 60 * 1e3;
 var DEFAULT_MAX_ROOMS = 1e3;
 var DEFAULT_MAX_GLOBAL_EVENTS = 1e5;
 var DEFAULT_MAX_GLOBAL_BYTES = 64 * 1024 * 1024;
+var DEFAULT_SOURCE_KEY_LIMIT = 2e3;
 var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
 var ProjectRoomBrokerResourceCapacityError = class extends Error {
   constructor(resource) {
@@ -20523,6 +20753,7 @@ var ProjectRoomEventBroker = class {
   constructor(options = {}) {
     this.rooms = /* @__PURE__ */ new Map();
     this.sequenceScopes = /* @__PURE__ */ new Map();
+    this.seenSourceKeys = /* @__PURE__ */ new Map();
     this.eventCount = 0;
     this.byteCount = 0;
     this.subscriberCount = 0;
@@ -20550,6 +20781,10 @@ var ProjectRoomEventBroker = class {
       options.maxGlobalBytes ?? DEFAULT_MAX_GLOBAL_BYTES,
       "maxGlobalBytes"
     );
+    this.sourceKeyLimit = positiveSafeInteger(
+      options.sourceKeyLimit ?? DEFAULT_SOURCE_KEY_LIMIT,
+      "sourceKeyLimit"
+    );
     this.sequence = nonnegativeSafeInteger(options.initialSequence ?? 0, "initialSequence");
     this.log = options.log ?? { warn: () => void 0 };
   }
@@ -20561,8 +20796,22 @@ var ProjectRoomEventBroker = class {
   get retainedBytes() {
     return this.byteCount;
   }
-  /** Atomically stores an immutable event before queueing detached listener callbacks. */
-  publish(scope, draft) {
+  /**
+   * Installs (or clears) a hook invoked after a locally published event is stored
+   * and queued to local subscribers. Used to mirror events to other replicas via
+   * a shared transport. Best-effort: a throwing hook never affects `publish`.
+   */
+  setBroadcastHook(hook) {
+    this.broadcast = hook;
+  }
+  /**
+   * Atomically stores an immutable event before queueing detached listener callbacks.
+   *
+   * @param sourceKey Optional caller-owned fact key; when present it is retained
+   * for bounded cross-replica duplicate suppression so a later foreign copy of the
+   * same fact (or a local publish) is dropped by {@link ingestForeignEvent}.
+   */
+  publish(scope, draft, sourceKey) {
     this.assertOpen();
     const canonicalScope = snapshotScope(scope);
     const key = scopeKey2(canonicalScope);
@@ -20597,9 +20846,52 @@ var ProjectRoomEventBroker = class {
     this.sequence = nextSequence;
     this.eventCount += 1;
     this.byteCount += bytes;
+    if (sourceKey !== void 0) this.rememberSourceKey(sourceKey);
     for (const subscription of [...room.subscriptions]) this.queueCallback(subscription, serialized);
+    if (this.broadcast) {
+      try {
+        this.broadcast(canonicalScope, serialized, sourceKey);
+      } catch {
+      }
+    }
     return parseScopedEvent(serialized);
   }
+  /**
+   * Delivers one event published by ANOTHER replica to this process's local
+   * subscribers for its room, without adding it to the local replay ring or
+   * consuming a local sequence (its id carries the origin replica's epoch).
+   *
+   * @param serializedEvent A serialized scoped business event produced by a peer.
+   * @param sourceKey Optional caller-owned fact key shared by every replica's copy
+   * of the same committed fact. When it was already seen (locally published or
+   * ingested from a peer) the event is dropped to avoid N× duplicate delivery.
+   * @returns True when it was queued to at least one local subscription.
+   * @remarks Source-key deduplication is process-local and bounded, so it caps
+   * duplicate fan-out within the retention window without replacing REST, which
+   * remains the authority. A malformed payload or a room with no local
+   * subscribers is ignored.
+   */
+  ingestForeignEvent(serializedEvent, sourceKey) {
+    this.assertOpen();
+    if (sourceKey !== void 0 && this.seenSourceKeys.has(sourceKey)) return false;
+    let key;
+    try {
+      const event = parseScopedEvent(serializedEvent);
+      key = scopeKey2(snapshotScope(event.scope));
+    } catch {
+      return false;
+    }
+    if (sourceKey !== void 0) this.rememberSourceKey(sourceKey);
+    const room = this.rooms.get(key);
+    if (!room || room.subscriptions.size === 0) return false;
+    this.touch(room);
+    let queued = false;
+    for (const subscription of [...room.subscriptions]) {
+      this.queueCallback(subscription, serializedEvent);
+      queued = true;
+    }
+    return queued;
+  }
   /** Atomically resolves replay and registers a distinct subscription for live events. */
   subscribe(scope, afterEventId, listener) {
     this.assertOpen();
@@ -20639,6 +20931,7 @@ var ProjectRoomEventBroker = class {
     }
     this.rooms.clear();
     this.sequenceScopes.clear();
+    this.seenSourceKeys.clear();
     this.eventCount = 0;
     this.byteCount = 0;
     this.subscriberCount = 0;
@@ -20773,6 +21066,14 @@ var ProjectRoomEventBroker = class {
     this.usageClock += 1;
     room.lastUsed = this.usageClock;
   }
+  rememberSourceKey(key) {
+    this.seenSourceKeys.set(key, true);
+    while (this.seenSourceKeys.size > this.sourceKeyLimit) {
+      const oldest = this.seenSourceKeys.keys().next().value;
+      if (oldest === void 0) break;
+      this.seenSourceKeys.delete(oldest);
+    }
+  }
 };
 function positiveSafeInteger(value, name) {
   if (!Number.isSafeInteger(value) || value <= 0) throw new RangeError(`${name} must be a positive safe integer`);
@@ -20970,7 +21271,7 @@ var BrokerProjectRoomRealtimePublisher = class {
       if (!event) throw new Error("Invalid Project Room realtime projection");
       const dedupeKey = createHash9("sha256").update(JSON.stringify([type, event.scope.tenantId, event.scope.projectId, event.scope.roomId, event.sourceKey])).digest("hex");
       if (this.sourceKeys.has(dedupeKey)) return;
-      this.deps.broker.publish(event.scope, event.draft);
+      this.deps.broker.publish(event.scope, event.draft, event.sourceKey);
       this.sourceKeys.set(dedupeKey, true);
       while (this.sourceKeys.size > this.dedupeLimit) {
         const oldest = this.sourceKeys.keys().next().value;
@@ -22374,7 +22675,7 @@ var registerLatticeRoutes = (app2, channelDeps, options = {}) => {
     bundleStore: getStoreLattice17("default", "capabilityBundle").store,
     projectStore
   }, { authRequired: process.env.AUTH_REQUIRED === "true" }));
-  registerModelProviderRoutes(app2, new ModelProviderController({
+  const modelProviderController = new ModelProviderController({
     providerStore: getStoreLattice17("default", "modelProvider").store,
     userTenantLinkStore,
     registry: {
@@ -22383,7 +22684,9 @@ var registerLatticeRoutes = (app2, channelDeps, options = {}) => {
       remove: (tenantId, modelKey2) => modelLatticeManager3.removeLatticeWithTenant(tenantId, modelKey2),
       listRegistered: (tenantId) => modelLatticeManager3.getLatticeKeysByTenant(tenantId)
     }
-  }, { authRequired: process.env.AUTH_REQUIRED === "true" }));
+  }, { authRequired: process.env.AUTH_REQUIRED === "true" });
+  registerModelProviderRoutes(app2, modelProviderController);
+  void modelProviderController.hydrateRegistry().catch(() => void 0);
   registerAgentWebAppRoutes(app2, {
     service: new AgentWebAppService({
       webAppStore,
@@ -22559,13 +22862,17 @@ var registerLatticeRoutes = (app2, channelDeps, options = {}) => {
       }) : void 0;
       const admitted = transferred ?? task;
       if (!admitted.workspaceId || !admitted.projectId) return;
+      const markerUpdated = await taskStore.update(admitted.tenantId, admitted.id, {
+        context: withExecutionRequest(admitted.context, (/* @__PURE__ */ new Date()).toISOString())
+      }).catch(() => null);
+      const eventTask = markerUpdated ?? admitted;
       void projectTaskAdmissionNotifier.publish({
-        tenantId: admitted.tenantId,
-        workspaceId: admitted.workspaceId,
-        projectId: admitted.projectId,
-        taskId: admitted.id,
-        ownerId: admitted.ownerId,
-        updatedAt: admitted.updatedAt.toISOString()
+        tenantId: eventTask.tenantId,
+        workspaceId: eventTask.workspaceId,
+        projectId: eventTask.projectId,
+        taskId: eventTask.id,
+        ownerId: eventTask.ownerId,
+        updatedAt: eventTask.updatedAt.toISOString()
       });
     }
   });
@@ -24800,13 +25107,25 @@ import {
   getLoggerLattice,
   loggerLatticeManager,
   sandboxLatticeManager as sandboxLatticeManager2,
-  getStoreLattice as getStoreLattice28,
+  getStoreLattice as getStoreLattice29,
   agentInstanceManager as agentInstanceManager9,
   createSandboxProvider,
   TokenCache,
   mcpManager as mcpManager3,
   convertToConnection as convertToConnection2,
-  createTrustedAgentMessageDispatcher as createTrustedAgentMessageDispatcher2
+  createTrustedAgentMessageDispatcher as createTrustedAgentMessageDispatcher2,
+  runAsLeader,
+  configureCoordinationLeaseStore,
+  getReplicaId as getReplicaId6,
+  taskLifecycleNotifier as taskLifecycleNotifier3,
+  projectTaskAdmissionNotifier as projectTaskAdmissionNotifier2,
+  scheduleLatticeManager as scheduleLatticeManager2,
+  OutboxPoller,
+  eventBus as eventBus4,
+  agentLatticeManager as agentLatticeManager5,
+  configurePluginCatalogSync,
+  PLUGIN_CATALOG_CHANGED_TOPIC,
+  getSandboxPluginCatalog as getSandboxPluginCatalog4
 } from "@axiom-lattice/core";
 
 // src/channels/ChannelInfrastructure.ts
@@ -25475,9 +25794,167 @@ async function prepareProjectTaskRuntime(runtime, log2) {
   }
 }
 
+// src/services/multiReplicaReadiness.ts
+import { getStoreLattice as getStoreLattice28 } from "@axiom-lattice/core";
+var SHARED_STORE_KINDS = ["lease", "outbox", "threadMessageQueue", "task", "taskWorkItem"];
+function storeClassName(kind) {
+  try {
+    return getStoreLattice28("default", kind).store.constructor?.name;
+  } catch {
+    return void 0;
+  }
+}
+function warnMultiReplicaReadiness(logger7) {
+  if (typeof process.env.AXIOM_REPLICA_ID !== "string" || process.env.AXIOM_REPLICA_ID.trim() === "") return;
+  for (const kind of SHARED_STORE_KINDS) {
+    const name = storeClassName(kind);
+    if (name === void 0 || !name.startsWith("InMemory")) continue;
+    logger7.warn(`Multi-replica: store '${kind}' is ${name} (not shared); configure a shared store`, {
+      phase: "multi_replica_readiness",
+      store: kind
+    });
+  }
+  if (!process.env.AUTH_TOKEN_SECRET?.trim() && !process.env.JWT_SECRET?.trim()) {
+    logger7.warn(
+      "Multi-replica: AUTH_TOKEN_SECRET/JWT_SECRET unset; console sessions are signed per-process and will not verify across replicas",
+      { phase: "multi_replica_readiness", env: "AUTH_TOKEN_SECRET" }
+    );
+  }
+  if (!process.env.LATTICE_ENCRYPTION_KEY?.trim()) {
+    logger7.warn(
+      "Multi-replica: LATTICE_ENCRYPTION_KEY unset; stored credentials use the shared dev default and may not decrypt across differing keys",
+      { phase: "multi_replica_readiness", env: "LATTICE_ENCRYPTION_KEY" }
+    );
+  }
+  logger7.warn(
+    "Multi-replica: agent run SSE and resume_stream are process-local; route a thread's follow-up runs to the replica streaming it (sticky routing) or share the stream/queue state",
+    { phase: "multi_replica_readiness", feature: "agent-run-sse" }
+  );
+}
+
+// src/services/OutboxRetentionJob.ts
+var HOUR_MS = 60 * 60 * 1e3;
+var DEFAULT_INTERVAL_MS2 = 6 * HOUR_MS;
+var DEFAULT_INITIAL_DELAY_MS2 = 6e4;
+var AXIOM_OUTBOX_RETENTION_HOURS_ENV = "AXIOM_OUTBOX_RETENTION_HOURS";
+var DEFAULT_RETENTION_HOURS = 24;
+function parseRetentionHours(value) {
+  if (value === void 0 || value === null) return DEFAULT_RETENTION_HOURS;
+  if (typeof value !== "string") return void 0;
+  const trimmed = value.trim();
+  if (trimmed.length === 0) return DEFAULT_RETENTION_HOURS;
+  const parsed = Number(trimmed);
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : void 0;
+}
+var OutboxRetentionJob = class {
+  /** @param deps Outbox slice, logger, and optional retention/interval bounds. */
+  constructor(deps) {
+    this.deps = deps;
+    this.running = false;
+    this.stopped = false;
+  }
+  /** Schedules one delayed sweep plus the recurring interval (both unref'd). */
+  start() {
+    if (this.stopped || this.deps.retentionMs === void 0) return;
+    if (this.initial !== void 0 || this.interval !== void 0) return;
+    this.initial = setTimeout(() => {
+      this.initial = void 0;
+      void this.runOnce();
+    }, this.deps.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS2);
+    this.initial.unref?.();
+    this.interval = setInterval(() => void this.runOnce(), this.deps.intervalMs ?? DEFAULT_INTERVAL_MS2);
+    this.interval.unref?.();
+  }
+  /**
+   * Runs one age-based prune.
+   *
+   * @returns The number of rows removed (0 when disabled, overlapping, or on failure).
+   */
+  async runOnce() {
+    const retentionMs = this.deps.retentionMs;
+    if (this.stopped || retentionMs === void 0 || this.running) return 0;
+    this.running = true;
+    try {
+      const removed = await this.deps.outbox.pruneOlderThan(retentionMs);
+      if (removed > 0) this.deps.log.info("outbox pruned", { removed });
+      return removed;
+    } catch (error) {
+      this.deps.log.warn("outbox prune failed", {
+        error: error instanceof Error ? error.message : String(error)
+      });
+      return 0;
+    } finally {
+      this.running = false;
+    }
+  }
+  /** Stops the timers; an in-flight sweep finishes on its own. */
+  stop() {
+    this.stopped = true;
+    if (this.initial !== void 0) {
+      clearTimeout(this.initial);
+      this.initial = void 0;
+    }
+    if (this.interval !== void 0) {
+      clearInterval(this.interval);
+      this.interval = void 0;
+    }
+  }
+};
+function createOutboxRetentionJob(deps) {
+  const retentionMs = deps.retentionMs ?? resolveRetentionMs(process.env[AXIOM_OUTBOX_RETENTION_HOURS_ENV]);
+  return new OutboxRetentionJob({ ...deps, retentionMs });
+}
+function resolveRetentionMs(value) {
+  const hours = parseRetentionHours(value);
+  return hours === void 0 ? void 0 : hours * HOUR_MS;
+}
+
+// src/services/collaboration/roomEventSync.ts
+var ROOM_EVENT_TOPIC = "room-event";
+async function publishRoomEvent(outbox, owner, scope, serialized, sourceKey) {
+  try {
+    await outbox.append({
+      topic: ROOM_EVENT_TOPIC,
+      payload: { scope, serialized, sourceKey },
+      originReplicaId: owner
+    });
+  } catch {
+  }
+}
+function ingestRoomEvent(payload, broker) {
+  const record = payload;
+  const serialized = record?.serialized;
+  if (typeof serialized !== "string" || serialized.length === 0) return false;
+  const sourceKey = record?.sourceKey;
+  if (typeof sourceKey === "string" && sourceKey.length > 0) {
+    return broker.ingestForeignEvent(serialized, sourceKey);
+  }
+  return broker.ingestForeignEvent(serialized);
+}
+
+// src/services/assistantRegistrySync.ts
+var ASSISTANT_CHANGE_EVENTS = ["assistant:created", "assistant:updated", "assistant:deleted"];
+var ASSISTANT_CHANGED_TOPIC = "assistant-changed";
+function bridgeAssistantEvents(subscribe, outbox, owner) {
+  for (const event of ASSISTANT_CHANGE_EVENTS) {
+    subscribe(event, (data) => {
+      const tenantId = data?.tenantId;
+      if (typeof tenantId !== "string" || tenantId.length === 0) return;
+      void outbox.append({ topic: ASSISTANT_CHANGED_TOPIC, payload: { tenantId }, originReplicaId: owner }).catch(() => void 0);
+    });
+  }
+}
+async function handleAssistantChanged(payload, reload) {
+  const tenantId = payload?.tenantId;
+  if (typeof tenantId !== "string" || tenantId.length === 0) return;
+  await reload(tenantId);
+}
+
 // src/index.ts
 import {
-  LoggerType
+  LoggerType,
+  REPLICA_HEADER as REPLICA_HEADER3,
+  AFFINITY_ECHO_HEADER as AFFINITY_ECHO_HEADER3
 } from "@axiom-lattice/protocols";
 process.on("unhandledRejection", (reason, promise) => {
   console.error("\u672A\u5904\u7406\u7684Promise\u62D2\u7EDD:", reason);
@@ -25557,7 +26034,7 @@ app.register(cors, {
   origin: true,
   methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"],
   allowedHeaders: "*",
-  exposedHeaders: ["Content-Type", "Content-Disposition"],
+  exposedHeaders: ["Content-Type", "Content-Disposition", REPLICA_HEADER3, AFFINITY_ECHO_HEADER3],
   credentials: true
 });
 app.register(sensible);
@@ -25609,7 +26086,7 @@ function getConfiguredSandboxProvider() {
 }
 async function restoreMcpConnections() {
   try {
-    const storeLattice = getStoreLattice28("default", "mcp");
+    const storeLattice = getStoreLattice29("default", "mcp");
     const store = storeLattice.store;
     if (!store) {
       logger6.info("MCP store not configured, skipping connection restoration");
@@ -25620,25 +26097,16 @@ async function restoreMcpConnections() {
     );
     if (configs.length === 0) return;
     logger6.info(`Restoring ${configs.length} MCP server connections...`);
-    for (const config of configs) {
-      try {
-        const connection = convertToConnection2(config.config);
-        mcpManager3.registerServer(config.tenantId, config.key, connection, {
-          timeoutMs: config.config.timeout,
-          maxAttempts: config.config.retryAttempts,
-          selectedTools: config.selectedTools
-        });
-      } catch (err) {
-        logger6.warn(`Failed to prepare MCP server "${config.tenantId}/${config.key}"`, { error: err });
-      }
-    }
-    await mcpManager3.connectAll();
+    const deps = {
+      store,
+      manager: mcpManager3,
+      convertToConnection: convertToConnection2
+    };
     const results = await Promise.allSettled(
       configs.map(
-        (config) => mcpManager3.registerToolsToToolLattice(
-          config.tenantId,
-          config.key,
-          config.selectedTools
+        (config) => reconcileMcpConfigChange(
+          { tenantId: config.tenantId, serverKey: config.key },
+          deps
         )
       )
     );
@@ -25646,7 +26114,7 @@ async function restoreMcpConnections() {
       if (result.status === "rejected") {
         const config = configs[index];
         logger6.warn(
-          `Failed to register tools for MCP server "${config.tenantId}/${config.key}"`,
+          `Failed to restore MCP server "${config.tenantId}/${config.key}"`,
           { error: result.reason }
         );
       }
@@ -25673,9 +26141,9 @@ var start = async (config) => {
     let projectRoomRealtimeRuntime;
     if (projectRoomRealtimeMode === "single-gateway") {
       projectRoomRealtimeRuntime = createProjectRoomRealtimeRuntime({
-        tasks: getStoreLattice28("default", "task").store,
-        rooms: getStoreLattice28("default", "projectRoom").store,
-        memberships: getStoreLattice28("default", "projectBotMembership").store,
+        tasks: getStoreLattice29("default", "task").store,
+        rooms: getStoreLattice29("default", "projectRoom").store,
+        memberships: getStoreLattice29("default", "projectBotMembership").store,
         log: { warn: (message, context) => logger6.warn(message, context) }
       });
     }
@@ -25718,7 +26186,7 @@ var start = async (config) => {
     }
     setEvalRunService(evalRunner);
     try {
-      const menuStore = getStoreLattice28("default", "menu").store;
+      const menuStore = getStoreLattice29("default", "menu").store;
       setMenuRegistry(menuStore);
       logger6.info("Menu registry initialized");
     } catch {
@@ -25736,10 +26204,10 @@ var start = async (config) => {
       const { A2AAuthService, parseEnvKeys } = await import("./A2AAuthService-7UGQ7GMW.mjs");
       const { registerA2AStandardRoutes } = await import("./a2a-standard-PBYPNHOA.mjs");
       const a2a = await import("./a2a-FGYX2HH3.mjs");
-      const a2aKeyStore = getStoreLattice28("default", "a2aApiKey").store;
-      const assistantStore = getStoreLattice28("default", "assistant").store;
-      const projectStore = getStoreLattice28("default", "project").store;
-      const taskStore = getStoreLattice28("default", "task").store;
+      const a2aKeyStore = getStoreLattice29("default", "a2aApiKey").store;
+      const assistantStore = getStoreLattice29("default", "assistant").store;
+      const projectStore = getStoreLattice29("default", "project").store;
+      const taskStore = getStoreLattice29("default", "task").store;
       const authService = new A2AAuthService({
         keyStore: a2aKeyStore,
         assistantStore,
@@ -25764,22 +26232,22 @@ var start = async (config) => {
       });
     }
     try {
-      const { registerOpenRoutes } = await import("./open-STFMNRO5.mjs");
-      const { registerOpenSessionRoutes } = await import("./open-session-IHDGAXL3.mjs");
+      const { registerOpenRoutes } = await import("./open-UVKMONVE.mjs");
+      const { registerOpenSessionRoutes } = await import("./open-session-2Y7XJJW2.mjs");
       const { registerOpenCatalogRoutes } = await import("./open-catalog-SZQ435AY.mjs");
-      const { registerOpenFilesRoutes } = await import("./open-files-NPNQQ6TY.mjs");
+      const { registerOpenFilesRoutes } = await import("./open-files-X3HXVEEM.mjs");
       const { registerOpenAuditRoutes } = await import("./open-audit-UJ3NXCGW.mjs");
       const { registerOpenExposureRoutes } = await import("./open-exposure-VU227IO3.mjs");
       const { openLattice } = await import("./openLattice-Q6M5S6CY.mjs");
       const { createKbProvider } = await import("./kbProvider-2D6DZ34N.mjs");
-      const { createOpenSandboxProvider } = await import("./sandboxProvider-I3P66VXT.mjs");
+      const { createOpenSandboxProvider } = await import("./sandboxProvider-DUYWHYJN.mjs");
       const { createAgentProvider } = await import("./agentProvider-X4FVQPKW.mjs");
       const { registerExposedPluginTools } = await import("./registerExposedPlugins-6GCOE64C.mjs");
       const { CollectionService: CollectionService2 } = await import("./collection_service-RTRJEQFL.mjs");
       openLattice.register(createKbProvider(new CollectionService2()));
       openLattice.register(createOpenSandboxProvider());
-      const threadStore = getStoreLattice28("default", "thread").store;
-      const assistantStore = getStoreLattice28("default", "assistant").store;
+      const threadStore = getStoreLattice29("default", "thread").store;
+      const assistantStore = getStoreLattice29("default", "assistant").store;
       openLattice.register(
         createAgentProvider({
           threadStore,
@@ -25802,13 +26270,24 @@ var start = async (config) => {
         error: err instanceof Error ? err.message : String(err)
       });
     }
+    try {
+      const { registerGatewaySandboxOpenCredentialProvider } = await import("./sandboxOpenCredentialProvider-USIU7Z7H.mjs");
+      registerGatewaySandboxOpenCredentialProvider((error) => {
+        logger6.warn("sandbox open-credential provider failed", { error });
+      });
+      logger6.info("Sandbox open-credential provider registered");
+    } catch (err) {
+      logger6.warn("Sandbox open-credential provider unavailable", {
+        error: err instanceof Error ? err.message : String(err)
+      });
+    }
     if (!sandboxLatticeManager2.hasLattice("default")) {
       sandboxLatticeManager2.registerLattice("default", getConfiguredSandboxProvider());
       logger6.info("Registered sandbox manager from env configuration");
     }
     try {
-      const { ResourceController } = await import("./resources-DXPXAXEZ.mjs");
-      const sharedResourceStore = getStoreLattice28("default", "sharedResource").store;
+      const { ResourceController } = await import("./resources-2L3KSUUU.mjs");
+      const sharedResourceStore = getStoreLattice29("default", "sharedResource").store;
       const cache = new TokenCache();
       const resourceController = new ResourceController({
         store: sharedResourceStore,
@@ -25837,30 +26316,98 @@ var start = async (config) => {
       }
     }
     const target_port = config?.port || Number(process.env.PORT) || 4001;
-    if (routes.projectTaskRuntime) {
-      await prepareProjectTaskRuntime(routes.projectTaskRuntime, logger6);
+    configureCoordinationLeaseStore(getStoreLattice29("default", "lease").store);
+    const outboxStore = getStoreLattice29("default", "outbox").store;
+    taskLifecycleNotifier3.setOutbox(outboxStore);
+    projectTaskAdmissionNotifier2.setOutbox(outboxStore);
+    configurePluginCatalogSync({ outbox: outboxStore, owner: getReplicaId6() });
+    bridgeAssistantEvents(
+      (event, cb) => eventBus4.subscribe(event, cb),
+      outboxStore,
+      getReplicaId6()
+    );
+    if (projectRoomRealtimeRuntime) {
+      projectRoomRealtimeRuntime.broker.setBroadcastHook((scope, serialized, sourceKey) => {
+        void publishRoomEvent(outboxStore, getReplicaId6(), scope, serialized, sourceKey);
+      });
     }
-    if (process.env.AXIOM_TASK_ORPHAN_RECOVERY === "true") {
-      try {
-        const { createOrphanRecoveryRuntime: createOrphanRecoveryRuntime2 } = await import("./OrphanRecoveryRuntime-MQDEJX3M.mjs");
-        const { createRecoveryCallerDeps: createRecoveryCallerDeps2 } = await import("./RecoveryCallerDeps-6SJUYJ5S.mjs");
-        const taskStore = getStoreLattice28("default", "task").store;
-        const workItemStore = getStoreLattice28("default", "taskWorkItem").store;
-        const tenantStore = getStoreLattice28("default", "tenant").store;
-        const runtime = createOrphanRecoveryRuntime2({
-          taskStore,
-          workItemStore,
-          tenantStore,
-          ...createRecoveryCallerDeps2(),
-          log: logger6
-        });
-        await runtime.recoverAll();
-      } catch (err) {
-        logger6.warn("Task orphan recovery failed", { error: err instanceof Error ? err.message : String(err) });
+    const outboxPoller = new OutboxPoller({
+      store: outboxStore,
+      owner: getReplicaId6(),
+      handlers: {
+        "task-lifecycle": (payload) => taskLifecycleNotifier3.deliverFromOutbox(payload),
+        "project-task-admission": (payload) => projectTaskAdmissionNotifier2.deliverFromOutbox(payload),
+        [ROOM_EVENT_TOPIC]: async (payload) => {
+          if (projectRoomRealtimeRuntime) ingestRoomEvent(payload, projectRoomRealtimeRuntime.broker);
+        },
+        "assistant-changed": (payload) => handleAssistantChanged(payload, (tenantId) => agentLatticeManager5.initializeStoredAssistantsForTenant(tenantId)),
+        [MCP_CONFIG_CHANGED_TOPIC]: (payload) => reconcileMcpConfigChange(payload, {
+          store: getStoreLattice29("default", "mcp").store,
+          manager: mcpManager3,
+          convertToConnection: convertToConnection2
+        }),
+        [PLUGIN_CATALOG_CHANGED_TOPIC]: async (payload) => {
+          const tenantId = payload?.tenantId;
+          if (typeof tenantId === "string" && tenantId.length > 0) getSandboxPluginCatalog4().invalidate(tenantId);
+        }
+      },
+      log: logger6
+    });
+    try {
+      await outboxPoller.start();
+    } catch (err) {
+      logger6.warn("Outbox poller failed to start", { error: err instanceof Error ? err.message : String(err) });
+    }
+    const leaseStore = getStoreLattice29("default", "lease").store;
+    if (leaseStore.constructor?.name === "InMemoryLeaseStore") {
+      logger6.warn(
+        "Startup recovery leader gate uses a non-distributed lease store (per-process only); configure a shared 'lease' store for multi-replica deployments",
+        { phase: "startup_recovery" }
+      );
+    }
+    warnMultiReplicaReadiness(logger6);
+    const startup = await runAsLeader(leaseStore, "startup-recovery", async () => {
+      if (routes.projectTaskRuntime) {
+        await prepareProjectTaskRuntime(routes.projectTaskRuntime, logger6);
       }
+      if (process.env.AXIOM_TASK_ORPHAN_RECOVERY === "true") {
+        try {
+          const { createOrphanRecoveryRuntime: createOrphanRecoveryRuntime2 } = await import("./OrphanRecoveryRuntime-BEFILJMU.mjs");
+          const { createRecoveryCallerDeps: createRecoveryCallerDeps2 } = await import("./RecoveryCallerDeps-6SJUYJ5S.mjs");
+          const runtime = createOrphanRecoveryRuntime2({
+            taskStore: getStoreLattice29("default", "task").store,
+            workItemStore: getStoreLattice29("default", "taskWorkItem").store,
+            tenantStore: getStoreLattice29("default", "tenant").store,
+            ...createRecoveryCallerDeps2(),
+            log: logger6
+          });
+          await runtime.recoverAll();
+        } catch (err) {
+          logger6.warn("Task orphan recovery failed", { error: err instanceof Error ? err.message : String(err) });
+        }
+      }
+      if (scheduleLatticeManager2.hasLattice("default")) {
+        try {
+          const restored = await scheduleLatticeManager2.getScheduleLattice("default").restore();
+          logger6.info("Scheduled tasks restored", { phase: "startup_recovery", restored });
+        } catch (err) {
+          logger6.warn("Scheduled task restore failed", {
+            phase: "startup_recovery",
+            error: err instanceof Error ? err.message : String(err)
+          });
+        }
+      }
+    }, { hold: true, ttlMs: 5 * 6e4 });
+    if (!startup.ran) {
+      logger6.info("Startup recovery skipped (another replica leads)", { phase: "startup_recovery" });
     }
     await app.listen({ port: target_port, host: "0.0.0.0" });
     logger6.info(`Lattice Gateway is running on port: ${target_port}`);
+    const outboxRetention = createOutboxRetentionJob({
+      outbox: outboxStore,
+      log: logger6
+    });
+    outboxRetention.start();
     const queueServiceConfig = config?.queueServiceConfig;
     if (queueServiceConfig) {
       setQueueServiceType(queueServiceConfig.type);
@@ -25886,8 +26433,13 @@ var LatticeGateway = {
 };
 export {
   LatticeGateway,
+  ProjectRoomEventBroker,
+  ROOM_EVENT_TOPIC,
   createOrphanRecoveryRuntime,
   createRecoveryCallerDeps,
+  createTaskRunLiveness,
+  ingestRoomEvent,
+  publishRoomEvent,
   recoverTask,
   recoverTopCaller,
   resolveTopCaller,