clay-server 2.34.0-beta.3 → 2.34.0-beta.5

package/lib/yoke/adapters/claude-worker.js

@@ -31,6 +31,7 @@ var abortController = null;
 var pendingPermissions = {};  // requestId -> resolve
 var pendingAskUser = {};      // toolUseId -> resolve
 var pendingElicitations = {}; // requestId -> resolve
+var pendingMcpToolCalls = {}; // requestId -> { resolve, reject }
 var conn = null;
 var buffer = "";
 
@@ -81,6 +82,90 @@ function getSDK() {
   return sdkModule;
 }
 
+function buildZodShape(z, inputSchema) {
+  if (!inputSchema || !inputSchema.properties) return {};
+  var shape = {};
+  var props = inputSchema.properties;
+  var required = inputSchema.required || [];
+  var keys = Object.keys(props);
+
+  for (var i = 0; i < keys.length; i++) {
+    var key = keys[i];
+    var prop = props[key];
+    var field;
+
+    if (prop.type === "number" || prop.type === "integer") {
+      field = z.number();
+    } else if (prop.type === "boolean") {
+      field = z.boolean();
+    } else if (prop.type === "array") {
+      field = z.array(z.any());
+    } else if (prop.type === "object") {
+      field = z.record(z.any());
+    } else if (prop.enum) {
+      field = z.enum(prop.enum);
+    } else {
+      field = z.string();
+    }
+
+    if (prop.description) field = field.describe(prop.description);
+    if (required.indexOf(key) === -1) field = field.optional();
+    shape[key] = field;
+  }
+
+  return shape;
+}
+
+function createWorkerMcpToolHandler(serverName, toolName) {
+  return function(args) {
+    var requestId = crypto.randomUUID();
+    sendToDaemon({
+      type: "mcp_tool_call",
+      requestId: requestId,
+      serverName: serverName,
+      toolName: toolName,
+      args: args || {},
+    });
+    return new Promise(function(resolve, reject) {
+      pendingMcpToolCalls[requestId] = { resolve: resolve, reject: reject };
+    });
+  };
+}
+
+function buildMcpServersFromDescriptors(descriptors, sdk) {
+  if (!descriptors || !descriptors.length) return null;
+  var z;
+  try { z = require("zod").z; } catch (e) {
+    try { z = require("zod"); } catch (e2) { return null; }
+  }
+
+  var servers = {};
+  for (var i = 0; i < descriptors.length; i++) {
+    var descriptor = descriptors[i];
+    if (!descriptor || !descriptor.serverName || !descriptor.tools || !descriptor.tools.length) continue;
+    var tools = [];
+    for (var j = 0; j < descriptor.tools.length; j++) {
+      var toolDescriptor = descriptor.tools[j];
+      if (!toolDescriptor || !toolDescriptor.name) continue;
+      tools.push(sdk.tool(
+        toolDescriptor.name,
+        toolDescriptor.description || toolDescriptor.name,
+        buildZodShape(z, toolDescriptor.inputSchema),
+        createWorkerMcpToolHandler(descriptor.serverName, toolDescriptor.name)
+      ));
+    }
+    if (tools.length > 0) {
+      servers[descriptor.serverName] = sdk.createSdkMcpServer({
+        name: descriptor.serverName,
+        version: "1.0.0",
+        tools: tools,
+      });
+    }
+  }
+
+  return Object.keys(servers).length > 0 ? servers : null;
+}
+
 // --- IPC helpers ---
 function sendToDaemon(msg) {
   if (!conn || conn.destroyed) return;
@@ -127,6 +212,9 @@ function handleMessage(msg) {
     case "elicitation_response":
       handleElicitationResponse(msg);
       break;
+    case "mcp_tool_result":
+      handleMcpToolResult(msg);
+      break;
     case "warmup":
       handleWarmup(msg);
       break;
@@ -208,6 +296,17 @@ function handleElicitationResponse(msg) {
   }
 }
 
+function handleMcpToolResult(msg) {
+  var pending = pendingMcpToolCalls[msg.requestId];
+  if (!pending) return;
+  delete pendingMcpToolCalls[msg.requestId];
+  if (msg.error) {
+    pending.reject(new Error(msg.error));
+    return;
+  }
+  pending.resolve(msg.result);
+}
+
 // --- Query handling ---
 async function handleQueryStart(msg) {
   var t0 = msg._perfT0 || Date.now();
@@ -238,6 +337,15 @@ async function handleQueryStart(msg) {
   options.abortController = abortController;
   options.debug = true;
   options.debugFile = "/tmp/clay-cli-debug-" + process.pid + ".log";
+  if (options.mcpServerDescriptors && options.mcpServerDescriptors.length) {
+    try {
+      var mcpServers = buildMcpServersFromDescriptors(options.mcpServerDescriptors, sdk);
+      if (mcpServers) options.mcpServers = mcpServers;
+    } catch (e) {
+      console.error("[sdk-worker] Failed to build MCP servers:", e.message || e);
+    }
+    delete options.mcpServerDescriptors;
+  }
   // Override CLI subprocess spawn to inject NODE_OPTIONS for IPv4-first DNS.
   // The SDK constructs its own env for the CLI process, so worker env vars
   // like NODE_OPTIONS are not inherited. We intercept the spawn to fix this.

package/lib/yoke/adapters/claude.js

@@ -648,7 +648,7 @@ function cleanupWorker(worker) {
 // in-process QueryHandle. This allows processQueryStream to iterate a worker
 // query identically to an in-process query.
 
-function createWorkerQueryHandle(worker, canUseTool, onElicitation) {
+function createWorkerQueryHandle(worker, canUseTool, onElicitation, callMcpTool) {
   // Async iterable state
   var iterQueue = [];
   var iterWaiting = null;
@@ -741,6 +741,20 @@ function createWorkerQueryHandle(worker, canUseTool, onElicitation) {
         }
         break;
 
+      case "mcp_tool_call":
+        if (callMcpTool) {
+          callMcpTool(msg.serverName, msg.toolName, msg.args || {}).then(function(result) {
+            worker.send({ type: "mcp_tool_result", requestId: msg.requestId, result: result });
+          }).catch(function(e) {
+            worker.send({
+              type: "mcp_tool_result",
+              requestId: msg.requestId,
+              error: (e && e.message) ? e.message : String(e),
+            });
+          });
+        }
+        break;
+
       case "context_usage":
       case "model_changed":
       case "effort_changed":
@@ -1231,7 +1245,7 @@ function createClaudeAdapter(opts) {
     }
 
     // Create the worker query handle (sets up message handler on worker)
-    var handle = createWorkerQueryHandle(worker, queryOpts.canUseTool, queryOpts.onElicitation);
+    var handle = createWorkerQueryHandle(worker, queryOpts.canUseTool, queryOpts.onElicitation, queryOpts.callMcpTool);
 
     // Wait for worker to be ready before sending query_start
     if (!reusingWorker) {
@@ -1254,7 +1268,7 @@ function createClaudeAdapter(opts) {
     if (claudeOpts.allowDangerouslySkipPermissions) queryOptions.allowDangerouslySkipPermissions = true;
     if (claudeOpts.settings) queryOptions.settings = claudeOpts.settings;
 
-    if (queryOpts.toolServers) queryOptions.mcpServers = queryOpts.toolServers;
+    if (queryOpts.toolServerDescriptors) queryOptions.mcpServerDescriptors = queryOpts.toolServerDescriptors;
     if (queryOpts.model) queryOptions.model = queryOpts.model;
     if (queryOpts.effort) queryOptions.effort = queryOpts.effort;
     if (queryOpts.resumeSessionId) queryOptions.resume = queryOpts.resumeSessionId;

package/lib/yoke/adapters/codex.js

@@ -89,6 +89,56 @@ function generateUuid() {
   return "codex-" + ts + "-" + cnt + "-" + rnd;
 }
 
+function waitMs(ms) {
+  return new Promise(function(resolve) {
+    setTimeout(resolve, ms);
+  });
+}
+
+function waitForProcessExit(proc, timeoutMs) {
+  return new Promise(function(resolve) {
+    if (!proc) {
+      resolve(true);
+      return;
+    }
+
+    if (proc.exitCode !== null || proc.signalCode !== null) {
+      resolve(true);
+      return;
+    }
+
+    var done = false;
+    var timer = null;
+
+    function cleanup() {
+      if (done) return;
+      done = true;
+      if (timer) clearTimeout(timer);
+      proc.removeListener("exit", onDone);
+      proc.removeListener("close", onDone);
+    }
+
+    function onDone() {
+      cleanup();
+      resolve(true);
+    }
+
+    proc.once("exit", onDone);
+    proc.once("close", onDone);
+
+    timer = setTimeout(function() {
+      cleanup();
+      resolve(false);
+    }, timeoutMs || 5000);
+  });
+}
+
+function createShutdownError() {
+  var err = new Error("Codex adapter is shutting down, retry shortly");
+  err.code = "CODEX_ADAPTER_SHUTTING_DOWN";
+  return err;
+}
+
 function normalizePlanStatus(status) {
   if (status === "inProgress") return "in_progress";
   if (status === "completed") return "completed";
@@ -310,16 +360,33 @@ function flattenEvent(notification, state) {
         state.thinkingBlocks[item.id] = "blk_" + state.blockCounter;
         events.push({ yokeType: "thinking_start", blockId: "blk_" + state.blockCounter });
       }
-      if (item.text) {
+      // Codex reasoning items may expose plain text via `text`, a short
+      // `summary`, or nested `content` parts. Prefer whichever is present;
+      // many turns arrive with only encrypted reasoning and no readable
+      // text at all, in which case the UI will hide the expand affordance.
+      var reasoningText = "";
+      if (typeof item.text === "string" && item.text.length > 0) {
+        reasoningText = item.text;
+      } else if (typeof item.summary === "string" && item.summary.length > 0) {
+        reasoningText = item.summary;
+      } else if (Array.isArray(item.content)) {
+        var parts = [];
+        for (var rpi = 0; rpi < item.content.length; rpi++) {
+          var rp = item.content[rpi];
+          if (rp && typeof rp.text === "string") parts.push(rp.text);
+        }
+        reasoningText = parts.join("\n");
+      }
+      if (reasoningText) {
         var thinkBlockId = state.thinkingBlocks[item.id];
         var prevThinkLen = state.thinkingLengths[item.id] || 0;
-        if (item.text.length > prevThinkLen) {
+        if (reasoningText.length > prevThinkLen) {
           events.push({
             yokeType: "thinking_delta",
             blockId: thinkBlockId,
-            text: item.text.substring(prevThinkLen),
+            text: reasoningText.substring(prevThinkLen),
           });
-          state.thinkingLengths[item.id] = item.text.length;
+          state.thinkingLengths[item.id] = reasoningText.length;
         }
       }
       if (evtPhase === "completed") {
@@ -503,6 +570,7 @@ function createCodexQueryHandle(appServer, queryOpts) {
   var systemPrompt = queryOpts.systemPrompt || "";
   var canUseTool = queryOpts.canUseTool || null;
   var onElicitation = queryOpts.onElicitation || null;
+  var onFinished = queryOpts.onFinished || null;
 
   // Check if the query was cancelled (either via handle.abort() or direct signal abort)
   function isCancelled() {
@@ -533,6 +601,19 @@ function createCodexQueryHandle(appServer, queryOpts) {
   var eventBuffer = [];
   var eventWaiting = null;
   var iteratorDone = false;
+  var finishedNotified = false;
+
+  function notifyFinished() {
+    if (finishedNotified) return;
+    finishedNotified = true;
+    if (typeof onFinished === "function") {
+      try {
+        onFinished();
+      } catch (e) {
+        console.error("[yoke/codex] onFinished error:", e.message || e);
+      }
+    }
+  }
 
   function pushEvent(evt) {
     if (iteratorDone) return;
@@ -552,6 +633,7 @@ function createCodexQueryHandle(appServer, queryOpts) {
       eventWaiting = null;
       resolve({ value: undefined, done: true });
     }
+    notifyFinished();
   }
 
   // Message queue for multi-turn
@@ -936,48 +1018,182 @@ function createCodexQueryHandle(appServer, queryOpts) {
 
 function createCodexAdapter(opts) {
   var _cwd = (opts && opts.cwd) || process.cwd();
+  var _slug = (opts && opts.slug) || "";
+  var _defaultInitOpts = Object.assign({}, opts || {});
   var _cachedModels = [];
   var _appServer = null;
   var _initPromise = null;
-  var _initOpts = null; // stored for query-time access
+  var _shutdownPromise = null;
+  var _refCount = 0;
+  var _lastActiveAt = Date.now();
+  var _shuttingDown = false;
+  var _activeQueries = [];
+
+  function updateLastActiveAt() {
+    _lastActiveAt = Date.now();
+  }
+
+  function registerActiveQuery(entry) {
+    _activeQueries.push(entry);
+  }
+
+  function removeActiveQuery(entry) {
+    var next = [];
+    for (var i = 0; i < _activeQueries.length; i++) {
+      if (_activeQueries[i] !== entry) next.push(_activeQueries[i]);
+    }
+    _activeQueries = next;
+  }
+
+  function decrementRefCount() {
+    if (_refCount > 0) {
+      _refCount--;
+    } else {
+      console.error("[yoke/codex] refCount negative, bug!");
+      _refCount = 0;
+    }
+    updateLastActiveAt();
+  }
+
+  function buildReadyResponse(skillNames) {
+    return {
+      models: _cachedModels,
+      defaultModel: "gpt-5.4",
+      skills: skillNames || [],
+      slashCommands: skillNames || [],
+      fastModeState: null,
+      capabilities: {
+        thinking: true,
+        betas: false,
+        rewind: false,
+        sessionResume: true,
+        promptSuggestions: true,
+        elicitation: true,
+        fileCheckpointing: false,
+        contextCompacting: false,
+        toolPolicy: ["ask", "allow-all"],
+      },
+    };
+  }
+
+  function clearRuntimeState() {
+    _appServer = null;
+    _initPromise = null;
+    _cachedModels = [];
+    _refCount = 0;
+    _activeQueries = [];
+    updateLastActiveAt();
+  }
+
+  function waitForRefCount(targetCount, timeoutMs) {
+    var deadline = Date.now() + (timeoutMs || 5000);
+    return new Promise(function(resolve) {
+      function tick() {
+        if (_refCount <= targetCount) {
+          resolve(true);
+          return;
+        }
+        if (Date.now() >= deadline) {
+          resolve(false);
+          return;
+        }
+        setTimeout(tick, 50);
+      }
+      tick();
+    });
+  }
+
+  function stopAppServer(deadlineMs) {
+    var proc = _appServer && _appServer.proc ? _appServer.proc : null;
+    if (!_appServer) return Promise.resolve(true);
+    try {
+      _appServer.stop();
+    } catch (e) {
+      console.error("[yoke/codex] App-server stop error:", e.message || e);
+    }
+    if (!proc) return Promise.resolve(true);
+    var remaining = (typeof deadlineMs === "number") ? Math.max(0, deadlineMs - Date.now()) : 5000;
+    return waitForProcessExit(proc, remaining).then(function(exited) {
+      if (!exited) {
+        try {
+          proc.kill("SIGKILL");
+        } catch (e) {}
+      }
+      return exited;
+    });
+  }
+
+  function beginShutdown(force, idleMs) {
+    if (_shutdownPromise) return _shutdownPromise;
+    if (_shuttingDown) return null;
+
+    _shuttingDown = true;
+
+    _shutdownPromise = (async function() {
+      var deadline = Date.now() + 5000;
+      var shouldAbort = !!force;
+
+      if (_initPromise) {
+        try {
+          await Promise.race([
+            _initPromise.catch(function() { return null; }),
+            waitMs(Math.max(0, deadline - Date.now())),
+          ]);
+        } catch (e) {}
+      }
+
+      if (shouldAbort && _activeQueries.length > 0) {
+        var active = _activeQueries.slice();
+        for (var i = 0; i < active.length; i++) {
+          try {
+            if (active[i] && active[i].abort) active[i].abort();
+          } catch (e) {}
+        }
+        await waitForRefCount(0, Math.max(0, deadline - Date.now()));
+      }
+
+      if (_appServer) {
+        await stopAppServer(deadline);
+      }
+
+      clearRuntimeState();
+      _shuttingDown = false;
+      _shutdownPromise = null;
+      return true;
+    })().catch(function(err) {
+      clearRuntimeState();
+      _shuttingDown = false;
+      _shutdownPromise = null;
+      throw err;
+    });
+
+    return _shutdownPromise;
+  }
 
   var adapter = {
     vendor: "codex",
 
     init: function(initOpts) {
+      if (_shuttingDown) {
+        return Promise.reject(createShutdownError());
+      }
+
+      var effectiveInitOpts = Object.assign({}, _defaultInitOpts, initOpts || {});
+
       // Already initialized - return cached result
       if (_appServer && _appServer.started && _cachedModels.length > 0) {
-        return Promise.resolve({
-          models: _cachedModels,
-          defaultModel: "gpt-5.4",
-          skills: [],
-          slashCommands: [],
-          fastModeState: null,
-          capabilities: {
-            thinking: true,
-            betas: false,
-            rewind: false,
-            sessionResume: true,
-            promptSuggestions: true,
-            elicitation: true,
-            fileCheckpointing: false,
-            contextCompacting: false,
-            toolPolicy: ["ask", "allow-all"],
-          },
-        });
+        return Promise.resolve(buildReadyResponse([]));
       }
 
       // Deduplicate concurrent init calls
       if (_initPromise) return _initPromise;
 
       _initPromise = (async function() {
-        _initOpts = initOpts;
-
         var serverOpts = { cwd: _cwd };
 
         // Extract adapter options
-        if (initOpts && initOpts.adapterOptions && initOpts.adapterOptions.CODEX) {
-          var co = initOpts.adapterOptions.CODEX;
+        if (effectiveInitOpts && effectiveInitOpts.adapterOptions && effectiveInitOpts.adapterOptions.CODEX) {
+          var co = effectiveInitOpts.adapterOptions.CODEX;
           if (co.apiKey) serverOpts.env = Object.assign({}, serverOpts.env || {}, { OPENAI_API_KEY: co.apiKey });
           if (co.baseUrl) serverOpts.env = Object.assign({}, serverOpts.env || {}, { OPENAI_BASE_URL: co.baseUrl });
           if (co.config) serverOpts.config = co.config;
@@ -1005,10 +1221,10 @@ function createCodexAdapter(opts) {
 
         // Track 2: Add clay-tools bridge server for in-app + remote MCP tools.
         var bridgePath = require("path").join(__dirname, "..", "mcp-bridge-server.js");
-        var clayPort = (initOpts && initOpts.clayPort) || process.env.CLAY_PORT || 2633;
-        var clayTls = (initOpts && initOpts.clayTls) || false;
-        var clayAuthToken = (initOpts && initOpts.clayAuthToken) || "";
-        var claySlug = (initOpts && initOpts.slug) || "";
+        var clayPort = effectiveInitOpts.clayPort || process.env.CLAY_PORT || 2633;
+        var clayTls = effectiveInitOpts.clayTls || false;
+        var clayAuthToken = effectiveInitOpts.clayAuthToken || "";
+        var claySlug = effectiveInitOpts.slug || _slug || "";
         try {
           if (require("fs").existsSync(bridgePath)) {
             var bridgeArgs = [bridgePath, "--port", String(clayPort), "--slug", claySlug];
@@ -1049,6 +1265,11 @@ function createCodexAdapter(opts) {
         });
         _appServer.notify("initialized", {});
 
+        if (_shuttingDown) {
+          await stopAppServer(Date.now() + 1000);
+          throw createShutdownError();
+        }
+
         console.log("[codex] App-server initialized, models: gpt-5.4, gpt-5.4-mini, gpt-5.3-codex, gpt-5.3-codex-spark, gpt-5.2");
 
         _cachedModels = [
@@ -1097,26 +1318,15 @@ function createCodexAdapter(opts) {
           console.error("[codex] Failed to discover skills:", e.message);
         }
 
+        if (_shuttingDown) {
+          await stopAppServer(Date.now() + 1000);
+          throw createShutdownError();
+        }
+
         _initPromise = null;
+        updateLastActiveAt();
 
-        return {
-          models: _cachedModels,
-          defaultModel: "gpt-5.4",
-          skills: skillNames,
-          slashCommands: skillNames,
-          fastModeState: null,
-          capabilities: {
-            thinking: true,
-            betas: false,
-            rewind: false,
-            sessionResume: true,
-            promptSuggestions: true,
-            elicitation: true,
-            fileCheckpointing: false,
-            contextCompacting: false,
-            toolPolicy: ["ask", "allow-all"],
-          },
-        };
+        return buildReadyResponse(skillNames);
       })();
 
       return _initPromise;
@@ -1134,12 +1344,31 @@ function createCodexAdapter(opts) {
     },
 
     createQuery: async function(queryOpts) {
+      if (_shuttingDown) {
+        throw createShutdownError();
+      }
+
+      if (!_appServer || !_appServer.started) {
+        await adapter.init(queryOpts || {});
+      }
+
+      if (_shuttingDown) {
+        throw createShutdownError();
+      }
+
       if (!_appServer || !_appServer.started) {
         throw new Error("[yoke/codex] Adapter not initialized. Call init() first.");
       }
 
       var model = queryOpts.model || "gpt-5.4";
       var ac = queryOpts.abortController || new AbortController();
+      var activeEntry = {
+        abort: function() {
+          try {
+            ac.abort();
+          } catch (e) {}
+        },
+      };
 
       // Map YOKE options to Codex thread options
       var codexOpts = (queryOpts.adapterOptions && queryOpts.adapterOptions.CODEX) || {};
@@ -1176,7 +1405,33 @@ function createCodexAdapter(opts) {
 
       console.log("[yoke/codex] createQuery: model=" + model + " approval=" + handleOpts.approvalPolicy + " sandbox=" + handleOpts.sandboxMode);
 
-      var handle = createCodexQueryHandle(_appServer, handleOpts);
+      _refCount++;
+      registerActiveQuery(activeEntry);
+
+      var handle;
+      try {
+        handleOpts.onFinished = function() {
+          removeActiveQuery(activeEntry);
+          decrementRefCount();
+        };
+        handle = createCodexQueryHandle(_appServer, handleOpts);
+      } catch (e) {
+        removeActiveQuery(activeEntry);
+        decrementRefCount();
+        throw e;
+      }
+
+      activeEntry.handle = handle;
+      activeEntry.abort = function() {
+        try {
+          if (handle && typeof handle.abort === "function") {
+            handle.abort();
+          } else {
+            ac.abort();
+          }
+        } catch (e) {}
+      };
+
       return handle;
     },
 
@@ -1243,10 +1498,19 @@ function createCodexAdapter(opts) {
 
     // Shutdown the app-server process
     shutdown: function() {
-      if (_appServer) {
-        _appServer.stop();
-        _appServer = null;
-      }
+      return beginShutdown(true);
+    },
+
+    shutdownIfIdle: function(idleMs) {
+      if (_shuttingDown || _shutdownPromise) return Promise.resolve(false);
+      if (_initPromise) return Promise.resolve(false);
+      if (!_appServer) return Promise.resolve(false);
+      if (_refCount > 0) return Promise.resolve(false);
+      if (Date.now() - _lastActiveAt < (idleMs || 0)) return Promise.resolve(false);
+      return beginShutdown(false).then(function() {
+        console.log("[yoke/codex] Reclaimed idle adapter for project " + (_slug || _cwd));
+        return true;
+      });
     },
   };