clay-server 2.34.0-beta.2 → 2.34.0-beta.4

package/lib/sessions.js

@@ -680,35 +680,82 @@ function createSessionManager(opts) {
     var session = sessions.get(localId);
     if (!session) return { hits: [], total: 0 };
     var q = query.toLowerCase();
+    var qLen = query.length;
     var history = session.history;
     var hits = [];
-    var lastAssistantHitTurn = -1; // track current assistant turn to deduplicate delta hits
-    var currentTurnStart = -1;
+
+    // Assistant turns can consist of many streaming deltas (especially Codex,
+    // where agentMessage/delta fragments arrive in small chunks). We accumulate
+    // delta text per turn, scan for ALL occurrences of the query across the
+    // accumulated buffer, then map each occurrence back to the historyIndex of
+    // the delta that contains its starting offset. This catches multiple
+    // matches within a single turn and also matches that straddle delta
+    // boundaries.
+    var turnBuffer = "";
+    var turnSegments = []; // [{ start, end, historyIndex, ts }]
+
+    function pushScalarHits(text, historyIndex, role, ts) {
+      if (!text) return;
+      var lower = text.toLowerCase();
+      var from = 0;
+      while (true) {
+        var idx = lower.indexOf(q, from);
+        if (idx === -1) break;
+        var s = Math.max(0, idx - 15);
+        var e = Math.min(text.length, idx + qLen + 15);
+        var snippet = (s > 0 ? "\u2026" : "") + text.substring(s, e) + (e < text.length ? "\u2026" : "");
+        hits.push({ historyIndex: historyIndex, snippet: snippet, role: role, ts: ts });
+        from = idx + qLen;
+      }
+    }
+
+    function flushTurn() {
+      if (!turnBuffer || turnSegments.length === 0) {
+        turnBuffer = "";
+        turnSegments = [];
+        return;
+      }
+      var lowerBuf = turnBuffer.toLowerCase();
+      var from = 0;
+      var segCursor = 0;
+      while (true) {
+        var idx = lowerBuf.indexOf(q, from);
+        if (idx === -1) break;
+        // Advance segCursor to the segment containing idx.
+        while (segCursor < turnSegments.length - 1 && turnSegments[segCursor].end <= idx) {
+          segCursor++;
+        }
+        var seg = turnSegments[segCursor];
+        var s = Math.max(0, idx - 15);
+        var e = Math.min(turnBuffer.length, idx + qLen + 15);
+        var snippet = (s > 0 ? "\u2026" : "") + turnBuffer.substring(s, e) + (e < turnBuffer.length ? "\u2026" : "");
+        hits.push({ historyIndex: seg.historyIndex, snippet: snippet, role: "assistant", ts: seg.ts });
+        from = idx + qLen;
+      }
+      turnBuffer = "";
+      turnSegments = [];
+    }
+
     for (var i = 0; i < history.length; i++) {
       var entry = history[i];
-      if (entry.type === "user_message" || entry.type === "mention_user") {
-        currentTurnStart = i;
-        lastAssistantHitTurn = -1;
-      }
-      if ((entry.type === "delta" || entry.type === "user_message" || entry.type === "mention_user" || entry.type === "mention_response" || entry.type === "debate_turn_done" || entry.type === "debate_comment_injected") && entry.text) {
-        // Skip duplicate delta hits within the same assistant turn
-        if (entry.type === "delta" && currentTurnStart === lastAssistantHitTurn) continue;
-        var text = entry.text;
-        var lowerText = text.toLowerCase();
-        var idx = lowerText.indexOf(q);
-        if (idx === -1) continue;
-        var start = Math.max(0, idx - 15);
-        var end = Math.min(text.length, idx + query.length + 15);
-        var snippet = (start > 0 ? "\u2026" : "") + text.substring(start, end) + (end < text.length ? "\u2026" : "");
-        if (entry.type === "delta") lastAssistantHitTurn = currentTurnStart;
-        hits.push({
+      var t = entry.type;
+      if (t === "user_message" || t === "mention_user") {
+        flushTurn();
+        pushScalarHits(entry.text, i, t === "user_message" ? "user" : "assistant", entry._ts || null);
+      } else if (t === "delta" && entry.text) {
+        turnSegments.push({
+          start: turnBuffer.length,
+          end: turnBuffer.length + entry.text.length,
           historyIndex: i,
-          snippet: snippet,
-          role: entry.type === "user_message" ? "user" : "assistant",
           ts: entry._ts || null,
         });
+        turnBuffer += entry.text;
+      } else if ((t === "mention_response" || t === "debate_turn_done" || t === "debate_comment_injected") && entry.text) {
+        flushTurn();
+        pushScalarHits(entry.text, i, "assistant", entry._ts || null);
       }
     }
+    flushTurn();
     return { hits: hits, total: history.length };
   }
 

package/lib/yoke/adapters/codex.js

@@ -89,6 +89,56 @@ function generateUuid() {
   return "codex-" + ts + "-" + cnt + "-" + rnd;
 }
 
+function waitMs(ms) {
+  return new Promise(function(resolve) {
+    setTimeout(resolve, ms);
+  });
+}
+
+function waitForProcessExit(proc, timeoutMs) {
+  return new Promise(function(resolve) {
+    if (!proc) {
+      resolve(true);
+      return;
+    }
+
+    if (proc.exitCode !== null || proc.signalCode !== null) {
+      resolve(true);
+      return;
+    }
+
+    var done = false;
+    var timer = null;
+
+    function cleanup() {
+      if (done) return;
+      done = true;
+      if (timer) clearTimeout(timer);
+      proc.removeListener("exit", onDone);
+      proc.removeListener("close", onDone);
+    }
+
+    function onDone() {
+      cleanup();
+      resolve(true);
+    }
+
+    proc.once("exit", onDone);
+    proc.once("close", onDone);
+
+    timer = setTimeout(function() {
+      cleanup();
+      resolve(false);
+    }, timeoutMs || 5000);
+  });
+}
+
+function createShutdownError() {
+  var err = new Error("Codex adapter is shutting down, retry shortly");
+  err.code = "CODEX_ADAPTER_SHUTTING_DOWN";
+  return err;
+}
+
 function normalizePlanStatus(status) {
   if (status === "inProgress") return "in_progress";
   if (status === "completed") return "completed";
@@ -310,16 +360,33 @@ function flattenEvent(notification, state) {
         state.thinkingBlocks[item.id] = "blk_" + state.blockCounter;
         events.push({ yokeType: "thinking_start", blockId: "blk_" + state.blockCounter });
       }
-      if (item.text) {
+      // Codex reasoning items may expose plain text via `text`, a short
+      // `summary`, or nested `content` parts. Prefer whichever is present;
+      // many turns arrive with only encrypted reasoning and no readable
+      // text at all, in which case the UI will hide the expand affordance.
+      var reasoningText = "";
+      if (typeof item.text === "string" && item.text.length > 0) {
+        reasoningText = item.text;
+      } else if (typeof item.summary === "string" && item.summary.length > 0) {
+        reasoningText = item.summary;
+      } else if (Array.isArray(item.content)) {
+        var parts = [];
+        for (var rpi = 0; rpi < item.content.length; rpi++) {
+          var rp = item.content[rpi];
+          if (rp && typeof rp.text === "string") parts.push(rp.text);
+        }
+        reasoningText = parts.join("\n");
+      }
+      if (reasoningText) {
         var thinkBlockId = state.thinkingBlocks[item.id];
         var prevThinkLen = state.thinkingLengths[item.id] || 0;
-        if (item.text.length > prevThinkLen) {
+        if (reasoningText.length > prevThinkLen) {
           events.push({
             yokeType: "thinking_delta",
             blockId: thinkBlockId,
-            text: item.text.substring(prevThinkLen),
+            text: reasoningText.substring(prevThinkLen),
           });
-          state.thinkingLengths[item.id] = item.text.length;
+          state.thinkingLengths[item.id] = reasoningText.length;
         }
       }
       if (evtPhase === "completed") {
@@ -503,6 +570,7 @@ function createCodexQueryHandle(appServer, queryOpts) {
   var systemPrompt = queryOpts.systemPrompt || "";
   var canUseTool = queryOpts.canUseTool || null;
   var onElicitation = queryOpts.onElicitation || null;
+  var onFinished = queryOpts.onFinished || null;
 
   // Check if the query was cancelled (either via handle.abort() or direct signal abort)
   function isCancelled() {
@@ -533,6 +601,19 @@ function createCodexQueryHandle(appServer, queryOpts) {
   var eventBuffer = [];
   var eventWaiting = null;
   var iteratorDone = false;
+  var finishedNotified = false;
+
+  function notifyFinished() {
+    if (finishedNotified) return;
+    finishedNotified = true;
+    if (typeof onFinished === "function") {
+      try {
+        onFinished();
+      } catch (e) {
+        console.error("[yoke/codex] onFinished error:", e.message || e);
+      }
+    }
+  }
 
   function pushEvent(evt) {
     if (iteratorDone) return;
@@ -552,6 +633,7 @@ function createCodexQueryHandle(appServer, queryOpts) {
       eventWaiting = null;
       resolve({ value: undefined, done: true });
     }
+    notifyFinished();
   }
 
   // Message queue for multi-turn
@@ -936,48 +1018,182 @@ function createCodexQueryHandle(appServer, queryOpts) {
 
 function createCodexAdapter(opts) {
   var _cwd = (opts && opts.cwd) || process.cwd();
+  var _slug = (opts && opts.slug) || "";
+  var _defaultInitOpts = Object.assign({}, opts || {});
   var _cachedModels = [];
   var _appServer = null;
   var _initPromise = null;
-  var _initOpts = null; // stored for query-time access
+  var _shutdownPromise = null;
+  var _refCount = 0;
+  var _lastActiveAt = Date.now();
+  var _shuttingDown = false;
+  var _activeQueries = [];
+
+  function updateLastActiveAt() {
+    _lastActiveAt = Date.now();
+  }
+
+  function registerActiveQuery(entry) {
+    _activeQueries.push(entry);
+  }
+
+  function removeActiveQuery(entry) {
+    var next = [];
+    for (var i = 0; i < _activeQueries.length; i++) {
+      if (_activeQueries[i] !== entry) next.push(_activeQueries[i]);
+    }
+    _activeQueries = next;
+  }
+
+  function decrementRefCount() {
+    if (_refCount > 0) {
+      _refCount--;
+    } else {
+      console.error("[yoke/codex] refCount negative, bug!");
+      _refCount = 0;
+    }
+    updateLastActiveAt();
+  }
+
+  function buildReadyResponse(skillNames) {
+    return {
+      models: _cachedModels,
+      defaultModel: "gpt-5.4",
+      skills: skillNames || [],
+      slashCommands: skillNames || [],
+      fastModeState: null,
+      capabilities: {
+        thinking: true,
+        betas: false,
+        rewind: false,
+        sessionResume: true,
+        promptSuggestions: true,
+        elicitation: true,
+        fileCheckpointing: false,
+        contextCompacting: false,
+        toolPolicy: ["ask", "allow-all"],
+      },
+    };
+  }
+
+  function clearRuntimeState() {
+    _appServer = null;
+    _initPromise = null;
+    _cachedModels = [];
+    _refCount = 0;
+    _activeQueries = [];
+    updateLastActiveAt();
+  }
+
+  function waitForRefCount(targetCount, timeoutMs) {
+    var deadline = Date.now() + (timeoutMs || 5000);
+    return new Promise(function(resolve) {
+      function tick() {
+        if (_refCount <= targetCount) {
+          resolve(true);
+          return;
+        }
+        if (Date.now() >= deadline) {
+          resolve(false);
+          return;
+        }
+        setTimeout(tick, 50);
+      }
+      tick();
+    });
+  }
+
+  function stopAppServer(deadlineMs) {
+    var proc = _appServer && _appServer.proc ? _appServer.proc : null;
+    if (!_appServer) return Promise.resolve(true);
+    try {
+      _appServer.stop();
+    } catch (e) {
+      console.error("[yoke/codex] App-server stop error:", e.message || e);
+    }
+    if (!proc) return Promise.resolve(true);
+    var remaining = (typeof deadlineMs === "number") ? Math.max(0, deadlineMs - Date.now()) : 5000;
+    return waitForProcessExit(proc, remaining).then(function(exited) {
+      if (!exited) {
+        try {
+          proc.kill("SIGKILL");
+        } catch (e) {}
+      }
+      return exited;
+    });
+  }
+
+  function beginShutdown(force, idleMs) {
+    if (_shutdownPromise) return _shutdownPromise;
+    if (_shuttingDown) return null;
+
+    _shuttingDown = true;
+
+    _shutdownPromise = (async function() {
+      var deadline = Date.now() + 5000;
+      var shouldAbort = !!force;
+
+      if (_initPromise) {
+        try {
+          await Promise.race([
+            _initPromise.catch(function() { return null; }),
+            waitMs(Math.max(0, deadline - Date.now())),
+          ]);
+        } catch (e) {}
+      }
+
+      if (shouldAbort && _activeQueries.length > 0) {
+        var active = _activeQueries.slice();
+        for (var i = 0; i < active.length; i++) {
+          try {
+            if (active[i] && active[i].abort) active[i].abort();
+          } catch (e) {}
+        }
+        await waitForRefCount(0, Math.max(0, deadline - Date.now()));
+      }
+
+      if (_appServer) {
+        await stopAppServer(deadline);
+      }
+
+      clearRuntimeState();
+      _shuttingDown = false;
+      _shutdownPromise = null;
+      return true;
+    })().catch(function(err) {
+      clearRuntimeState();
+      _shuttingDown = false;
+      _shutdownPromise = null;
+      throw err;
+    });
+
+    return _shutdownPromise;
+  }
 
   var adapter = {
     vendor: "codex",
 
     init: function(initOpts) {
+      if (_shuttingDown) {
+        return Promise.reject(createShutdownError());
+      }
+
+      var effectiveInitOpts = Object.assign({}, _defaultInitOpts, initOpts || {});
+
       // Already initialized - return cached result
       if (_appServer && _appServer.started && _cachedModels.length > 0) {
-        return Promise.resolve({
-          models: _cachedModels,
-          defaultModel: "gpt-5.4",
-          skills: [],
-          slashCommands: [],
-          fastModeState: null,
-          capabilities: {
-            thinking: true,
-            betas: false,
-            rewind: false,
-            sessionResume: true,
-            promptSuggestions: true,
-            elicitation: true,
-            fileCheckpointing: false,
-            contextCompacting: false,
-            toolPolicy: ["ask", "allow-all"],
-          },
-        });
+        return Promise.resolve(buildReadyResponse([]));
       }
 
       // Deduplicate concurrent init calls
       if (_initPromise) return _initPromise;
 
       _initPromise = (async function() {
-        _initOpts = initOpts;
-
         var serverOpts = { cwd: _cwd };
 
         // Extract adapter options
-        if (initOpts && initOpts.adapterOptions && initOpts.adapterOptions.CODEX) {
-          var co = initOpts.adapterOptions.CODEX;
+        if (effectiveInitOpts && effectiveInitOpts.adapterOptions && effectiveInitOpts.adapterOptions.CODEX) {
+          var co = effectiveInitOpts.adapterOptions.CODEX;
           if (co.apiKey) serverOpts.env = Object.assign({}, serverOpts.env || {}, { OPENAI_API_KEY: co.apiKey });
           if (co.baseUrl) serverOpts.env = Object.assign({}, serverOpts.env || {}, { OPENAI_BASE_URL: co.baseUrl });
           if (co.config) serverOpts.config = co.config;
@@ -1005,10 +1221,10 @@ function createCodexAdapter(opts) {
 
         // Track 2: Add clay-tools bridge server for in-app + remote MCP tools.
         var bridgePath = require("path").join(__dirname, "..", "mcp-bridge-server.js");
-        var clayPort = (initOpts && initOpts.clayPort) || process.env.CLAY_PORT || 2633;
-        var clayTls = (initOpts && initOpts.clayTls) || false;
-        var clayAuthToken = (initOpts && initOpts.clayAuthToken) || "";
-        var claySlug = (initOpts && initOpts.slug) || "";
+        var clayPort = effectiveInitOpts.clayPort || process.env.CLAY_PORT || 2633;
+        var clayTls = effectiveInitOpts.clayTls || false;
+        var clayAuthToken = effectiveInitOpts.clayAuthToken || "";
+        var claySlug = effectiveInitOpts.slug || _slug || "";
         try {
           if (require("fs").existsSync(bridgePath)) {
             var bridgeArgs = [bridgePath, "--port", String(clayPort), "--slug", claySlug];
@@ -1049,6 +1265,11 @@ function createCodexAdapter(opts) {
         });
         _appServer.notify("initialized", {});
 
+        if (_shuttingDown) {
+          await stopAppServer(Date.now() + 1000);
+          throw createShutdownError();
+        }
+
         console.log("[codex] App-server initialized, models: gpt-5.4, gpt-5.4-mini, gpt-5.3-codex, gpt-5.3-codex-spark, gpt-5.2");
 
         _cachedModels = [
@@ -1097,26 +1318,15 @@ function createCodexAdapter(opts) {
           console.error("[codex] Failed to discover skills:", e.message);
         }
 
+        if (_shuttingDown) {
+          await stopAppServer(Date.now() + 1000);
+          throw createShutdownError();
+        }
+
         _initPromise = null;
+        updateLastActiveAt();
 
-        return {
-          models: _cachedModels,
-          defaultModel: "gpt-5.4",
-          skills: skillNames,
-          slashCommands: skillNames,
-          fastModeState: null,
-          capabilities: {
-            thinking: true,
-            betas: false,
-            rewind: false,
-            sessionResume: true,
-            promptSuggestions: true,
-            elicitation: true,
-            fileCheckpointing: false,
-            contextCompacting: false,
-            toolPolicy: ["ask", "allow-all"],
-          },
-        };
+        return buildReadyResponse(skillNames);
       })();
 
       return _initPromise;
@@ -1134,12 +1344,31 @@ function createCodexAdapter(opts) {
     },
 
     createQuery: async function(queryOpts) {
+      if (_shuttingDown) {
+        throw createShutdownError();
+      }
+
+      if (!_appServer || !_appServer.started) {
+        await adapter.init(queryOpts || {});
+      }
+
+      if (_shuttingDown) {
+        throw createShutdownError();
+      }
+
       if (!_appServer || !_appServer.started) {
         throw new Error("[yoke/codex] Adapter not initialized. Call init() first.");
       }
 
       var model = queryOpts.model || "gpt-5.4";
       var ac = queryOpts.abortController || new AbortController();
+      var activeEntry = {
+        abort: function() {
+          try {
+            ac.abort();
+          } catch (e) {}
+        },
+      };
 
       // Map YOKE options to Codex thread options
       var codexOpts = (queryOpts.adapterOptions && queryOpts.adapterOptions.CODEX) || {};
@@ -1176,7 +1405,33 @@ function createCodexAdapter(opts) {
 
       console.log("[yoke/codex] createQuery: model=" + model + " approval=" + handleOpts.approvalPolicy + " sandbox=" + handleOpts.sandboxMode);
 
-      var handle = createCodexQueryHandle(_appServer, handleOpts);
+      _refCount++;
+      registerActiveQuery(activeEntry);
+
+      var handle;
+      try {
+        handleOpts.onFinished = function() {
+          removeActiveQuery(activeEntry);
+          decrementRefCount();
+        };
+        handle = createCodexQueryHandle(_appServer, handleOpts);
+      } catch (e) {
+        removeActiveQuery(activeEntry);
+        decrementRefCount();
+        throw e;
+      }
+
+      activeEntry.handle = handle;
+      activeEntry.abort = function() {
+        try {
+          if (handle && typeof handle.abort === "function") {
+            handle.abort();
+          } else {
+            ac.abort();
+          }
+        } catch (e) {}
+      };
+
       return handle;
     },
 
@@ -1243,10 +1498,19 @@ function createCodexAdapter(opts) {
 
     // Shutdown the app-server process
     shutdown: function() {
-      if (_appServer) {
-        _appServer.stop();
-        _appServer = null;
-      }
+      return beginShutdown(true);
+    },
+
+    shutdownIfIdle: function(idleMs) {
+      if (_shuttingDown || _shutdownPromise) return Promise.resolve(false);
+      if (_initPromise) return Promise.resolve(false);
+      if (!_appServer) return Promise.resolve(false);
+      if (_refCount > 0) return Promise.resolve(false);
+      if (Date.now() - _lastActiveAt < (idleMs || 0)) return Promise.resolve(false);
+      return beginShutdown(false).then(function() {
+        console.log("[yoke/codex] Reclaimed idle adapter for project " + (_slug || _cwd));
+        return true;
+      });
     },
   };