clay-server 5.2.1-beta.1 → 5.2.1-beta.2

package/lib/sdk-bridge.js

@@ -1413,7 +1413,7 @@ function createSDKBridge(opts) {
     guardedPush.exactProductDelivery = !!(beforePush && beforePush.exactProductDelivery);
     session._queryGeneration = (session._queryGeneration || 0) + 1;
     session._queryStarting = true;
-    var starting = startQueryInner(session, text, images, linuxUser, guardedPush, onAccepted, recoveryState);
+    var starting = startQueryInner(session, text, images, linuxUser, guardedPush, onAccepted, recoveryState, payload);
     if (recoveryState) recoveryState.starting = starting;
     try {
       var accepted = await starting;
@@ -1494,7 +1494,7 @@ function createSDKBridge(opts) {
     };
   }
 
-  async function startQueryInner(session, text, images, linuxUser, beforePush, onAccepted, recoveryState) {
+  async function startQueryInner(session, text, images, linuxUser, beforePush, onAccepted, recoveryState, admissionPayload) {
     var sessionRuntimeEnv = getRuntimeEnv(session);
     var resumedAtQueryStart = !!session.cliSessionId;
     async function ensureVendorReady(vendor) {
@@ -1939,6 +1939,10 @@ function createSDKBridge(opts) {
         if (waiting) session._providerRecoveryWaiting = queryGeneration;
         else delete session._providerRecoveryWaiting;
       },
+      onRecoveryDiagnostic: function(evidence) {
+        if (sm.sessions.get(session.localId) !== session || Number(session._sdkQueryGeneration || 0) !== queryGeneration) return;
+        console.info('[sdk-bridge] Codex recovery admission ' + JSON.stringify({ sessionId: session.localId, queryGeneration: queryGeneration, evidence: evidence }));
+      },
       onRecoveryWaiting: function(waiting) {
         if (!queryContextAvailable()) return;
         sendAndRecord(session, { type: 'informational', level: 'info', content: waiting
@@ -2050,7 +2054,16 @@ function createSDKBridge(opts) {
       session.queryInstance = null;
       session.messageQueue = null;
       session.abortController = null;
-      sendAndRecord(session, { type: "error", text: "Failed to start query: " + (e.message || e) });
+      var startupError = { type: "error", text: "Failed to start query: " + (e.message || e) };
+      if (e.code === 'CODEX_RECOVERY_UNAVAILABLE' || e.code === 'CODEX_RECOVERY_PROBE_UNAVAILABLE') {
+        // Keep undelivered input with the persisted failure, outside the live
+        // input queue. A subsequent explicit send must not replay this batch.
+        startupError.recoveryAdmission = { state: 'not_sent', messages: [admissionPayload].concat(session.pendingPush || []).filter(Boolean).map(function(message) {
+          return { text: message.text, images: message.images || null };
+        }) };
+        startupError.text += ' Messages received during this wait were also not sent; they remain retained with this conversation. Send the messages you want to retry explicitly.';
+      }
+      sendAndRecord(session, startupError);
       sendAndRecord(session, { type: "done", code: 1 });
       sm.broadcastSessionList();
       return false;

package/lib/yoke/adapters/codex.js

@@ -1480,6 +1480,7 @@ function createCodexAdapter(opts) {
   var _recoveryIdleShutdown = null;
   var _activeQueries = [];
   var recoveryAdmission = require("../codex-recovery-admission").attachRecoveryAdmission({
+    timeoutMs: opts.recoveryAdmissionTimeoutMs,
     state: function() { return { epoch: _shutdownEpoch, shuttingDown: _shuttingDown, retiring: _recoveryIdleShutdown === _shutdownEpoch,
       initializing: _initializing, refCount: _refCount, server: _appServer, entries: _activeQueries.slice() }; },
     shutdownError: createShutdownError,

package/lib/yoke/codex-app-server.js

@@ -237,7 +237,9 @@ CodexAppServer.prototype._handleMessage = function(msg) {
       delete this.pendingRequests[msg.id];
       if (pending.timer) clearTimeout(pending.timer);
       if (msg.error) {
-        pending.reject(new Error(msg.error.message || JSON.stringify(msg.error)));
+        var rpcError = new Error(msg.error.message || JSON.stringify(msg.error));
+        rpcError.code = msg.error.code;
+        pending.reject(rpcError);
       } else {
         pending.resolve(msg.result);
       }

package/lib/yoke/codex-idle-recovery.js

@@ -21,24 +21,50 @@ function terminalsAreIdle(result) {
   });
 }
 
+function observe(checkCurrent, reason, method, rpcCode, threadId) {
+  if (typeof checkCurrent.observe === 'function') checkCurrent.observe(reason, method, rpcCode, threadId);
+}
+
+async function probe(server, method, params, timeout, checkCurrent) {
+  observe(checkCurrent, 'probe_started', method, null, params.threadId);
+  try {
+    var result = await server.send(method, params, timeout);
+    observe(checkCurrent, 'probe_returned', method, null, params.threadId);
+    return result;
+  } catch (error) {
+    observe(checkCurrent, 'probe_failed', method, error && Number.isInteger(error.code) ? error.code : null, params.threadId);
+    throw error;
+  }
+}
+
+function probeFailure(error, method) {
+  if (error && (error.code === -32601 || error.code === -32602)) {
+    var unsupported = new Error('Codex cannot verify recovery safety: ' + method + ' is unavailable or incompatible. No message was sent to the runtime.');
+    unsupported.code = 'CODEX_RECOVERY_PROBE_UNAVAILABLE';
+    throw unsupported;
+  }
+  return false;
+}
+
 async function retireIdleQueries(entries, server, checkCurrent) {
   checkCurrent();
   var candidates = [];
   for (var i = 0; i < entries.length; i++) {
     var handle = entries[i].handle;
     var snapshot = handle && handle.getIdleRecoverySnapshot && handle.getIdleRecoverySnapshot();
-    if (!snapshot) return false;
+    if (!snapshot) { observe(checkCurrent, 'idle_input_snapshot_unavailable', null, null, handle && handle.getRecoveryThreadId && handle.getRecoveryThreadId()); return false; }
     candidates.push({ handle: handle, snapshot: snapshot });
   }
   var safe = await Promise.all(candidates.map(function(candidate) {
-    return server.send("thread/backgroundTerminals/list", {
+    return probe(server, "thread/backgroundTerminals/list", {
       threadId: candidate.snapshot.threadId,
-    }, 5000).then(terminalsAreIdle, function() { return false; });
+    }, 5000, checkCurrent).then(terminalsAreIdle, function(error) { return probeFailure(error, 'thread/backgroundTerminals/list'); });
   }));
   checkCurrent();
   for (var ci = 0; ci < candidates.length; ci++) {
     var current = candidates[ci].handle.getIdleRecoverySnapshot();
-    if (!safe[ci] || !current || current.revision !== candidates[ci].snapshot.revision) return false;
+    if (!safe[ci]) { observe(checkCurrent, 'background_state_not_proven_idle', null, null, candidates[ci].snapshot.threadId); return false; }
+    if (!current || current.revision !== candidates[ci].snapshot.revision) { observe(checkCurrent, 'input_changed_during_probe', null, null, candidates[ci].snapshot.threadId); return false; }
   }
   // No awaits between validation and closing every input. A new send either
   // already invalidated its snapshot or sees a closed input and resumes anew.
@@ -58,33 +84,64 @@ async function loadedThreads(server, checkCurrent) {
   var deadline = Date.now() + 5000;
   for (var page = 0; page < 20; page++) {
     var remaining = deadline - Date.now();
-    if (remaining <= 0) return null;
+    if (remaining <= 0) { observe(checkCurrent, 'loaded_list_deadline'); return null; }
     var result;
     try {
-      result = await server.send('thread/loaded/list', { cursor: cursor, limit: 100 }, remaining);
+      result = await probe(server, 'thread/loaded/list', { cursor: cursor, limit: 100 }, remaining, checkCurrent);
     } catch (error) {
       checkCurrent();
       if (!server.started) throw error;
+      probeFailure(error, 'thread/loaded/list');
       return null;
     }
     checkCurrent();
-    if (!result || !Array.isArray(result.data) || !result.data.every(function(id) { return typeof id === 'string'; })) return null;
+    if (!result || !Array.isArray(result.data) || !result.data.every(function(id) { return typeof id === 'string'; })) { observe(checkCurrent, 'loaded_list_invalid'); return null; }
     threads = threads.concat(result.data);
     if (!result.nextCursor) return threads;
-    if (result.nextCursor === cursor) return null;
+    if (result.nextCursor === cursor) { observe(checkCurrent, 'loaded_list_cursor_repeated'); return null; }
     cursor = result.nextCursor;
   }
+  observe(checkCurrent, 'loaded_list_page_limit');
   return null;
 }
 
 async function threadIsAbsent(server, threadId, checkCurrent) {
   var threads = await loadedThreads(server, checkCurrent);
+  if (threads && threads.indexOf(threadId) !== -1) observe(checkCurrent, 'target_still_loaded', null, null, threadId);
   return !!threads && threads.indexOf(threadId) === -1;
 }
 
 async function runtimeCanStop(server, retiredThreads, checkCurrent) {
   var threads = await loadedThreads(server, checkCurrent);
-  return !!threads && threads.every(function(threadId) { return retiredThreads.indexOf(threadId) !== -1; });
+  if (!threads) return false;
+  // The caller has no live handles and holds admission. Closed handles can
+  // leave loaded threads that unsubscribe cannot release; prove every such
+  // thread idle before stopping their shared process, including siblings.
+  for (var threadId of threads) {
+    if (retiredThreads.indexOf(threadId) === -1 && !await threadIsIdle(server, threadId, checkCurrent)) return false;
+  }
+  checkCurrent();
+  return true;
+}
+
+async function threadIsIdle(server, threadId, checkCurrent) {
+  var method = 'thread/read';
+  try {
+    var status = await probe(server, method, { threadId: threadId, includeTurns: false }, 5000, checkCurrent);
+    checkCurrent();
+    if (!status || !status.thread || status.thread.id !== threadId
+        || !status.thread.status || status.thread.status.type !== 'idle') {
+      observe(checkCurrent, status && status.thread && status.thread.id === threadId && status.thread.status && status.thread.status.type === 'active'
+        ? 'provider_reports_active' : 'provider_idle_not_proven', method, null, threadId);
+      return false;
+    }
+    method = 'thread/backgroundTerminals/list';
+    var terminals = await probe(server, method, { threadId: threadId }, 5000, checkCurrent);
+    checkCurrent();
+    var idle = terminalsAreIdle(terminals);
+    if (!idle) observe(checkCurrent, 'background_state_not_proven_idle', method, null, threadId);
+    return idle;
+  } catch (error) { checkCurrent(); if (!server.started) throw error; return probeFailure(error, method); }
 }
 
 async function releaseThread(entries, server, threadId, checkCurrent) {
@@ -99,18 +156,14 @@ async function releaseThread(entries, server, threadId, checkCurrent) {
   } else {
     // A closed SDK handle can leave a loaded provider thread behind. Only an
     // explicitly idle provider thread with verified terminal state may unload.
-    try {
-      var status = await server.send('thread/read', { threadId: threadId, includeTurns: false }, 5000);
-      checkCurrent();
-      if (!status || !status.thread || status.thread.id !== threadId
-          || !status.thread.status || status.thread.status.type !== 'idle') return false;
-      var terminals = await server.send('thread/backgroundTerminals/list', { threadId: threadId }, 5000);
-      checkCurrent();
-      if (!terminalsAreIdle(terminals)) return false;
-    } catch (error) { checkCurrent(); if (!server.started) throw error; return false; }
+    if (!await threadIsIdle(server, threadId, checkCurrent)) return false;
+  }
+  try {
+    var result = await probe(server, 'thread/unsubscribe', { threadId: threadId }, 5000, checkCurrent);
+    var status = result && result.status;
+    observe(checkCurrent, ['unsubscribed', 'notSubscribed', 'notLoaded'].indexOf(status) !== -1 ? 'unsubscribe_' + status : 'unsubscribe_status_unknown', 'thread/unsubscribe', null, threadId);
   }
-  try { await server.send('thread/unsubscribe', { threadId: threadId }, 5000); }
-  catch (error) { checkCurrent(); if (!server.started) throw error; return false; }
+  catch (error) { checkCurrent(); if (!server.started) throw error; return probeFailure(error, 'thread/unsubscribe'); }
   return threadIsAbsent(server, threadId, checkCurrent);
 }
 

package/lib/yoke/codex-recovery-admission.js

@@ -1,4 +1,5 @@
 var idleRecovery = require('./codex-idle-recovery');
+var crypto = require('crypto');
 
 // Serialize decisions, not whole model turns. Waiting for a target must release
 // the gate so an unrelated Driver can still answer its Worker's permissions.
@@ -57,6 +58,8 @@ function attachRecoveryAdmission(ctx) {
     handoff = tail;
     tail.then(function() { if (handoff === tail) handoff = null; });
     try {
+      checkCurrent.phase('admission_gate');
+      if (prior) checkCurrent.observe('admission_gate_pending');
       await idleRecovery.wait(prior || Promise.resolve(), checkCurrent);
       await idleRecovery.wait(cleanup, checkCurrent);
       checkSharedExit();
@@ -69,8 +72,10 @@ function attachRecoveryAdmission(ctx) {
   }
 
   async function releaseTarget(threadId, checkCurrent) {
+    checkCurrent.phase('isolated_target');
     var prior = targets.get(threadId);
     if (!prior) return true;
+    checkCurrent.observe('isolated_target_reserved');
     if (cleanupFailures.has(prior)) throw cleanupFailures.get(prior);
     await prior.retireIdleForRecovery(checkCurrent);
     await cleanup;
@@ -80,8 +85,19 @@ function attachRecoveryAdmission(ctx) {
   }
 
   async function prepareShared(threadId, checkCurrent) {
+    checkCurrent.phase('shared_target');
     var state = ctx.state();
-    if (state.initializing) return false;
+    if (state.initializing) { checkCurrent.observe('shared_runtime_initializing'); return false; }
+    // Independent conversations do not participate in target handoff. Probing
+    // every idle sibling first can consume the whole admission budget even
+    // when this target can be unloaded immediately.
+    if (state.server && state.entries.some(function(entry) {
+      return !entry.handle || entry.handle.getRecoveryThreadId() !== threadId;
+    })) {
+      if (await idleRecovery.releaseThread(state.entries, state.server, threadId, checkCurrent)) return true;
+      checkCurrent();
+    }
+    checkCurrent.phase('shared_idle_retirement');
     var retiredThreads = [];
     if (state.refCount > 0) {
       var snapshots = state.entries.map(function(entry) { return entry.handle && entry.handle.getIdleRecoverySnapshot(); });
@@ -91,17 +107,52 @@ function attachRecoveryAdmission(ctx) {
     }
     state = ctx.state();
     if (!state.server) return true;
+    checkCurrent.phase('shared_runtime_release');
     if (!state.refCount && await idleRecovery.runtimeCanStop(state.server, retiredThreads, checkCurrent)) {
       var proc = state.server.proc;
       var stopped = await ctx.stopIdle();
       if (stopped) { unconfirmedSharedProcess = proc || null; checkSharedExit(); }
       return stopped;
     }
+    checkCurrent.phase('shared_target');
     return idleRecovery.releaseThread(state.entries, state.server, threadId, checkCurrent);
   }
 
   async function waitUntilAvailable(queryOpts, action) {
-    var checkCurrent = checker(queryOpts);
+    var checkLifecycle = checker(queryOpts);
+    var startedAt = Date.now();
+    var deadline = startedAt + (ctx.timeoutMs || 15000);
+    var admitted = false;
+    var observations = new Set();
+    var lastEvidence = null;
+    var ended = false;
+    var phase = 'admission_gate';
+    function targetKey(threadId) {
+      return crypto.createHash('sha256').update(String(threadId || queryOpts.resumeSessionId || '')).digest('hex').slice(0, 16);
+    }
+    checkCurrent.phase = function(value) { phase = value; };
+    function checkCurrent() {
+      checkLifecycle();
+      if (!admitted && Date.now() >= deadline) {
+        var error = new Error('Tool recovery could not verify that this conversation is safe to resume. No message was sent to the runtime. Review your retained messages and send again when the runtime is available.');
+        error.code = 'CODEX_RECOVERY_UNAVAILABLE';
+        throw error;
+      }
+    }
+    // Fixed evidence labels only: never log thread contents, RPC payloads,
+    // commands, credentials, or provider error strings. Bound polling noise.
+    checkCurrent.observe = function(reason, method, rpcCode, threadId) {
+      if (ended) return;
+      var evidence = { reason: reason, phase: phase, targetKey: targetKey(threadId), method: method || null, rpcCode: Number.isInteger(rpcCode) ? rpcCode : null };
+      lastEvidence = evidence;
+      var key = JSON.stringify(evidence);
+      if (observations.has(key) || observations.size >= 24) return;
+      observations.add(key);
+      if (typeof queryOpts.onRecoveryDiagnostic === 'function') {
+        try { queryOpts.onRecoveryDiagnostic(Object.assign({ elapsedMs: Date.now() - startedAt }, evidence)); } catch (error) {}
+      }
+    };
+    checkCurrent.admit = function() { checkCurrent(); admitted = true; };
     var waiting = false;
     if (typeof queryOpts.onRecoveryAdmission === 'function') queryOpts.onRecoveryAdmission(true);
     try {
@@ -119,6 +170,10 @@ function attachRecoveryAdmission(ctx) {
         await idleRecovery.wait(new Promise(function(resolve) { setTimeout(resolve, 50); }), checkCurrent);
       }
     } finally {
+      ended = true;
+      if (typeof queryOpts.onRecoveryDiagnostic === 'function') {
+        try { queryOpts.onRecoveryDiagnostic({ reason: admitted ? 'admission_granted' : 'admission_ended_without_grant', phase: phase, targetKey: targetKey(), elapsedMs: Date.now() - startedAt, lastEvidence: lastEvidence }); } catch (error) {}
+      }
       if (typeof queryOpts.onRecoveryAdmission === 'function') queryOpts.onRecoveryAdmission(false);
     }
   }
@@ -127,7 +182,9 @@ function attachRecoveryAdmission(ctx) {
     return waitUntilAvailable(queryOpts, async function(checkCurrent) {
       if (!await releaseTarget(queryOpts.resumeSessionId, checkCurrent)) return false;
       if (!await prepareShared(queryOpts.resumeSessionId, checkCurrent)) return false;
-      checkCurrent();
+      // Once ownership is reserved, normal startup/cleanup owns cancellation.
+      // A deadline must not discard a handle being created after admission.
+      checkCurrent.admit();
       var runtime = ctx.createRuntime(queryOpts);
       runtimes.push(runtime);
       targets.set(queryOpts.resumeSessionId, runtime);
@@ -149,7 +206,11 @@ function attachRecoveryAdmission(ctx) {
     checkSharedExit: checkSharedExit,
     hasTarget: function(threadId) { return targets.has(threadId); },
     waitForTarget: function(queryOpts) {
-      return waitUntilAvailable(queryOpts, function(checkCurrent) { return releaseTarget(queryOpts.resumeSessionId, checkCurrent); });
+      return waitUntilAvailable(queryOpts, async function(checkCurrent) {
+        var released = await releaseTarget(queryOpts.resumeSessionId, checkCurrent);
+        if (released) checkCurrent.admit();
+        return released;
+      });
     },
     shutdown: function() {
       return Promise.all(runtimes.slice().map(cleanupRuntime)).then(function(results) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clay-server",
-  "version": "5.2.1-beta.1",
+  "version": "5.2.1-beta.2",
   "description": "Self-hosted team workspace for Claude Code and Codex. Multi-user, browser-based, with persistent AI mates.",
   "bin": {
     "clay-server": "./bin/cli.js"