couchbase 3.8.2 → 3.9.1

data/ext/couchbase/core/io/http_session_manager.hxx

@@ -17,20 +17,17 @@
 
 #pragma once
 
-#include <couchbase/build_config.hxx>
-
 #include "core/app_telemetry_meter.hxx"
 #include "core/config_listener.hxx"
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-#include "core/columnar/bootstrap_notification_subscriber.hxx"
-#endif
 #include "core/logger/logger.hxx"
+#include "core/logger/redaction.hxx"
 #include "core/metrics/meter_wrapper.hxx"
 #include "core/operations/http_noop.hxx"
 #include "core/service_type.hxx"
 #include "core/tls_context_provider.hxx"
 #include "core/tracing/noop_tracer.hxx"
 #include "core/tracing/tracer_wrapper.hxx"
+#include "core/utils/movable_function.hxx"
 #include "http_command.hxx"
 #include "http_context.hxx"
 #include "http_session.hxx"
@@ -38,26 +35,22 @@
 
 #include <gsl/narrow>
 
+#include <atomic>
 #include <chrono>
+#include <exception>
 #include <optional>
 #include <queue>
 #include <random>
+#include <utility>
+#include <vector>
 
 namespace couchbase::core::io
 {
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-class http_session_manager
-  : public std::enable_shared_from_this<http_session_manager>
-  , public config_listener
-  , public columnar::bootstrap_notification_subscriber
-{
-#else
 class http_session_manager
   : public std::enable_shared_from_this<http_session_manager>
   , public config_listener
 {
-#endif
 public:
   http_session_manager(std::string client_id,
                        asio::io_context& ctx,
@@ -75,6 +68,11 @@ public:
     tracer_ = std::move(tracer);
   }
 
+  [[nodiscard]] auto tracer() const -> std::shared_ptr<tracing::tracer_wrapper>
+  {
+    return tracer_;
+  }
+
   void set_meter(std::shared_ptr<metrics::meter_wrapper> meter)
   {
     meter_ = std::move(meter);
@@ -91,33 +89,11 @@ public:
     return config_.capabilities;
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  void notify_bootstrap_error(const impl::bootstrap_error& error) override
-  {
-    CB_LOG_DEBUG("Received bootstrap error notification. code={}, ec_message={}, message={}, "
-                 "allow_fast_fail={}.",
-                 error.ec.value(),
-                 error.ec.message(),
-                 error.error_message,
-                 allow_fast_fail_ ? "true" : "false");
-    if (allow_fast_fail_) {
-      std::scoped_lock bootstrap_error_lock(last_bootstrap_error_mutex_);
-      last_bootstrap_error_ = error;
-      drain_deferred_queue(last_bootstrap_error_.value());
-    }
-  }
-
-  void notify_bootstrap_success(const std::string& session_id) override
-  {
-    CB_LOG_DEBUG("Received successful bootstrap notification.  Session={}.", session_id);
-    std::scoped_lock bootstrap_error_lock(last_bootstrap_error_mutex_);
-    allow_fast_fail_ = false;
-    last_bootstrap_error_.reset();
-  }
-#endif
-
   void update_config(topology::configuration config) override
   {
+    // Idle sessions to nodes that have left the cluster, collected under the lock
+    // and stopped afterwards (stop() re-enters sessions_mutex_ via on_stop).
+    std::vector<std::shared_ptr<http_session>> evicted;
     {
       std::scoped_lock config_lock(config_mutex_, sessions_mutex_, next_index_mutex_);
       config_ = std::move(config);
@@ -125,26 +101,28 @@ public:
         next_index_ = 0;
       }
       for (auto& [type, sessions] : idle_sessions_) {
-        sessions.remove_if([&opts = options_, &cfg = config_](const auto& session) {
-          return session && !cfg.has_node(opts.network,
-                                          session->type(),
-                                          opts.enable_tls,
-                                          session->hostname(),
-                                          session->port());
+        sessions.remove_if([&opts = options_, &cfg = config_, &evicted](const auto& session) {
+          if (session && !cfg.has_node(opts.network,
+                                       session->type(),
+                                       opts.enable_tls,
+                                       session->hostname(),
+                                       session->port())) {
+            evicted.push_back(session);
+            return true;
+          }
+          return false;
         });
       }
     }
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    drain_deferred_queue({});
-#endif
-  }
-
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  void set_dispatch_timeout(const std::chrono::milliseconds timeout)
-  {
-    dispatch_timeout_ = timeout;
+    // Tear the evicted connections down now instead of leaving them open until
+    // the idle timer fires; the node is gone, so the socket is dead weight (and,
+    // against a proxy that has already dropped the far side, a stale handle).
+    for (const auto& session : evicted) {
+      asio::post(session->get_executor(), [session]() {
+        session->stop();
+      });
+    }
   }
-#endif
 
   void set_configuration(const topology::configuration& config, const cluster_options& options)
   {
@@ -160,18 +138,7 @@ public:
       options_ = options;
       next_index_ = next_index;
       config_ = config;
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-      if (!configured_) {
-        configured_ = true;
-      }
-      if (allow_fast_fail_) {
-        allow_fast_fail_ = false;
-      }
-#endif
     }
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    drain_deferred_queue({});
-#endif
   }
 
   void export_diag_info(diag::diagnostics_result& res)
@@ -218,31 +185,26 @@ public:
           const auto& hostname = node.hostname_for(options_.network);
           const auto& canonical_hostname = node.hostname;
           std::uint16_t canonical_port = node.port_or(type, options_.enable_tls, 0);
-          auto session = create_session(type,
-                                        node_details{
-                                          hostname,
-                                          port,
-                                          node.node_uuid,
-                                          canonical_hostname,
-                                          canonical_port,
-                                        });
-          if (session->is_connected()) {
-            std::scoped_lock lock(sessions_mutex_);
-            busy_sessions_[type].push_back(session);
+          // Created and listed pending in one section, as check_out does, so a close() either
+          // stops it or precedes it. Unlisted while its connect is pending, it would be invisible
+          // to a close() whose completion lets the io_context stop.
+          std::shared_ptr<http_session> session;
+          {
+            const std::scoped_lock lock(sessions_mutex_);
+            session = create_session(type,
+                                     node_details{
+                                       hostname,
+                                       port,
+                                       node.node_uuid,
+                                       canonical_hostname,
+                                       canonical_port,
+                                     },
+                                     generation_);
+            pending_sessions_[type].push_back(session);
           }
           operations::http_noop_request request{};
           request.type = type;
           request.timeout = timeout;
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-          auto cmd = std::make_shared<operations::http_command<operations::http_noop_request>>(
-            ctx_,
-            request,
-            tracer_,
-            meter_,
-            app_telemetry_meter_,
-            options_.default_timeout_for(request.type),
-            dispatch_timeout_);
-#else
           auto cmd = std::make_shared<operations::http_command<operations::http_noop_request>>(
             ctx_,
             request,
@@ -250,12 +212,13 @@ public:
             meter_,
             app_telemetry_meter_,
             options_.default_timeout_for(request.type));
-#endif
+          cmd->set_command_session(session);
           cmd->start(
             [start = std::chrono::steady_clock::now(),
              self = shared_from_this(),
              type,
-             cmd,
+             session,
+             weak_cmd = std::weak_ptr(cmd),
              handler = collector->build_reporter()](operations::http_noop_response&& resp) {
               diag::ping_state state = diag::ping_state::ok;
               std::optional<std::string> error{};
@@ -271,27 +234,27 @@ public:
                                           ec.message(),
                                           resp.ctx.http_status));
               }
-              auto remote_address = cmd->session_->remote_address();
+              auto remote_address = session->remote_address();
               // If not connected, the remote address will be empty.  Better to
               // give the user some context on the "attempted" remote address.
               if (remote_address.empty()) {
-                remote_address =
-                  fmt::format("{}:{}", cmd->session_->hostname(), cmd->session_->port());
+                remote_address = fmt::format("{}:{}", session->hostname(), session->port());
               }
               handler->report(
                 diag::endpoint_ping_info{ type,
-                                          cmd->session_->id(),
+                                          session->id(),
                                           std::chrono::duration_cast<std::chrono::microseconds>(
                                             std::chrono::steady_clock::now() - start),
                                           remote_address,
-                                          cmd->session_->local_address(),
+                                          session->local_address(),
                                           state,
                                           {},
                                           error });
-              self->check_in(type, cmd->session_);
+              if (const auto cmd = weak_cmd.lock(); cmd) {
+                self->check_in(type, cmd->session_for_check_in());
+              }
             });
 
-          cmd->set_command_session(session);
           if (!session->is_connected()) {
             connect_then_send(session, cmd, {}, true);
           } else {
@@ -318,9 +281,8 @@ public:
     node_details preferred_node{};
 
     if (preferred_node_address.empty() && !undesired_node_address.empty()) {
-      // This is currently only used by Columnar to avoid the node
-      // that was last used and to satisfy the requirement for retrying on a different randomly
-      // selected node (as opposed to any node we have an idle session to).
+      // No sticky node, but one to avoid. Retrying elsewhere requires a different node chosen
+      // at random, not merely any node this manager happens to hold an idle session to.
       if (auto n = pick_random_node(type, undesired_node_address); n.port != 0) {
         preferred_node_address = fmt::format("{}:{}", n.hostname, n.port);
         preferred_node = std::move(n);
@@ -344,41 +306,26 @@ public:
       return !s;
     });
     std::shared_ptr<http_session> session{};
-    while (!idle_sessions_[type].empty()) {
-      if (preferred_node_address.empty()) {
-        session = idle_sessions_[type].front();
-        idle_sessions_[type].pop_front();
-        if (session->reset_idle()) {
-          break;
-        }
-      } else {
-        auto ptr = std::find_if(idle_sessions_[type].begin(),
-                                idle_sessions_[type].end(),
-                                [&preferred_node_address,
-                                 &h = preferred_node.hostname,
-                                 &p = preferred_node.port](const auto& s) {
-                                  // Check for a match using both the unresolved hostname & IP
-                                  // address
-                                  return (s->remote_address() == preferred_node_address ||
-                                          (s->hostname() == h && s->port() == std::to_string(p)));
-                                });
-        if (ptr != idle_sessions_[type].end()) {
-          session = *ptr;
-          idle_sessions_[type].erase(ptr);
-          if (session->reset_idle()) {
-            break;
-          }
-        } else {
-          session = create_session(type, preferred_node);
-          break;
-        }
-      }
-      CB_LOG_TRACE(
-        "{} Idle timer has expired for \"{}:{}\".  Attempting to select another session.",
-        session->log_prefix(),
-        session->hostname(),
-        session->port());
-      session.reset();
+    // Only a reusable session leaves the idle list. One that is not has a stop queued, by its idle
+    // timer or by retire_sessions(), and stays listed until on_stop unlists it, so a close() before
+    // then still stops it and waits for it.
+    auto& idle = idle_sessions_[type];
+    const auto taken = std::find_if(
+      idle.begin(),
+      idle.end(),
+      [&preferred_node_address, &h = preferred_node.hostname, &p = preferred_node.port](
+        const auto& s) {
+        // Check for a match using both the unresolved hostname & IP address
+        const bool matches = preferred_node_address.empty() ||
+                             s->remote_address() == preferred_node_address ||
+                             (s->hostname() == h && s->port() == std::to_string(p));
+        return matches && is_reusable(*s);
+      });
+    if (taken != idle.end()) {
+      session = *taken;
+      idle.erase(taken);
+    } else if (!preferred_node_address.empty()) {
+      session = create_session(type, preferred_node, generation_);
     }
     if (!session) {
       auto node = preferred_node_address.empty() ? next_node(type)
@@ -386,7 +333,7 @@ public:
       if (node.port == 0) {
         return { errc::common::service_not_available, nullptr };
       }
-      session = create_session(type, node);
+      session = create_session(type, node, generation_);
     }
     if (session->is_connected()) {
       busy_sessions_[type].push_back(session);
@@ -415,17 +362,48 @@ public:
       return;
     }
     bool should_stop = false;
-    std::chrono::milliseconds idle_timeout{};
     {
-      std::scoped_lock lock(config_mutex_);
-      if (!session->keep_alive() || !config_.has_node(options_.network,
-                                                      session->type(),
-                                                      options_.enable_tls,
-                                                      session->hostname(),
-                                                      session->port())) {
+      // config_mutex_ guards config_, which set_configuration() writes without sessions_mutex_.
+      // sessions_mutex_ in the same section keeps update_config()'s eviction, which holds both,
+      // from landing between the has_node test and the publication: that would pool a connection
+      // to a node that has left the cluster. check_in must therefore never be called with
+      // sessions_mutex_ held.
+      const std::scoped_lock lock(config_mutex_, sessions_mutex_);
+      // stopped_ is set before stop()'s teardown runs on_stop, whose cleanup takes
+      // sessions_mutex_. Read under that lock, either the stop is seen here, or its cleanup runs
+      // after the publication below and removes the session again.
+      //
+      // stopping_ is a stop the session's owner has decided and not yet executed, claimed before
+      // this check_in can run: by a streaming body under its own mutex while reading_complete_ is
+      // false (http_session::read_some runs the body's completion, which sets it, before the
+      // stream-end handler).
+      if (session->is_stopped() || session->is_stopping()) {
+        return;
+      }
+      if (session->pool_generation() != generation_ || session->tls_epoch() != tls_epoch_ ||
+          !session->keep_alive() ||
+          !config_.has_node(options_.network,
+                            session->type(),
+                            options_.enable_tls,
+                            session->hostname(),
+                            session->port())) {
         should_stop = true;
       } else {
-        idle_timeout = options_.idle_http_connection_timeout;
+        // Under sessions_mutex_, as check_out's reset_idle() is. set_idle() binds the timer
+        // completion to the strand and never calls stop() inline, so on_stop cannot re-enter
+        // sessions_mutex_ here.
+        session->set_idle(options_.idle_http_connection_timeout);
+        idle_sessions_[type].push_back(session);
+        if (auto busy_it = busy_sessions_.find(type); busy_it != busy_sessions_.end()) {
+          busy_it->second.remove_if([id = session->id()](const auto& s) -> bool {
+            return !s || s->id() == id;
+          });
+        }
+        if (auto pend_it = pending_sessions_.find(type); pend_it != pending_sessions_.end()) {
+          pend_it->second.remove_if([id = session->id()](const auto& s) -> bool {
+            return !s || s->id() == id;
+          });
+        }
       }
     }
     if (should_stop) {
@@ -433,72 +411,107 @@ public:
         session->stop();
       });
     }
-    if (!session->is_stopped()) {
-      // set_idle() arms the idle timer via async_wait — it never calls stop() inline,
-      // so on_stop cannot re-enter sessions_mutex_ here. It must precede publication to
-      // idle_sessions_ so a concurrent check_out's reset_idle() finds a pending timer.
-      session->set_idle(idle_timeout);
-      CB_LOG_DEBUG("{} put HTTP session back to idle connections", session->log_prefix());
-      std::scoped_lock lock(sessions_mutex_);
-      idle_sessions_[type].push_back(session);
-      if (auto busy_it = busy_sessions_.find(type); busy_it != busy_sessions_.end()) {
-        busy_it->second.remove_if([id = session->id()](const auto& s) -> bool {
-          return !s || s->id() == id;
-        });
-      }
-      if (auto pend_it = pending_sessions_.find(type); pend_it != pending_sessions_.end()) {
-        pend_it->second.remove_if([id = session->id()](const auto& s) -> bool {
-          return !s || s->id() == id;
-        });
+    CB_LOG_DEBUG("{} put HTTP session back to idle connections", session->log_prefix());
+  }
+
+  // Stops the reuse of every session created before the call, for a credential that the connection
+  // carries, such as a client certificate. Idle sessions stop now. A busy or connecting session
+  // completes its request first; check_in then stops it, because its TLS epoch is stale.
+  void retire_sessions()
+  {
+    std::vector<std::shared_ptr<http_session>> retired;
+    {
+      const std::scoped_lock lock(sessions_mutex_);
+      ++tls_epoch_;
+      // Claimed but left listed until their stop runs, so a close() before then stops them and
+      // waits for them. check_out skips a claimed session, and on_stop unlists it.
+      for (const auto& [type, sessions] : idle_sessions_) {
+        for (const auto& s : sessions) {
+          if (s) {
+            s->mark_stopping();
+            retired.push_back(s);
+          }
+        }
       }
     }
+    // Stopped on its own strand, as close() does.
+    for (auto& s : retired) {
+      asio::post(s->get_executor(), [s]() {
+        s->stop();
+      });
+    }
   }
 
-  void close()
+  // on_stopped runs once every session listed at the call has been stopped and holds no
+  // read_some() call, on the strand of the last one, or before close() returns when none is listed.
+  // Until then the read_some() calls queued on a session hold the body that holds the session, and
+  // its on_stop cleanup holds this manager. Whoever stops the io_context must wait for it: a
+  // handler still queued then never runs, and the session, its body and this manager are never
+  // released. A do_read() the stop aborts is not waited for: its completion holds only the session,
+  // which destroying the io_context releases.
+  void close(utils::movable_function<void()> on_stopped = {})
   {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    drain_deferred_queue(errc::common::request_canceled);
-#endif
     std::map<service_type, std::list<std::shared_ptr<http_session>>> busy_sessions, idle_sessions,
       pending_sessions;
     {
+      // A new generation starts with the lists moved out. check_in and publish_busy refuse a
+      // session from an earlier one, so none is listed again; a connect callback stops its session
+      // instead. check_out creates sessions in the new generation.
       const std::scoped_lock lock(sessions_mutex_);
-      busy_sessions = std::move(busy_sessions_);
-      idle_sessions = std::move(idle_sessions_);
-      pending_sessions = std::move(pending_sessions_);
-    }
-    for (auto& [type, sessions] : idle_sessions) {
-      for (auto& s : sessions) {
-        if (s) {
-          s->reset_idle();
-          s.reset();
+      ++generation_;
+      // Exchanged rather than moved: the manager goes on serving, and a moved-from map is only
+      // valid, not empty.
+      busy_sessions = std::exchange(busy_sessions_, {});
+      idle_sessions = std::exchange(idle_sessions_, {});
+      pending_sessions = std::exchange(pending_sessions_, {});
+    }
+    // Every session is stopped on its own strand, as check_in and update_config() do. stop()
+    // closes the stream and cancels the response, which the strand's read and write handlers use.
+    // Run on this thread, it races those handlers, and a queued do_write() finds no socket. An
+    // idle session is stopped rather than only reset_idle()d, so the read armed by set_idle() is
+    // torn down and its shared_ptr does not keep the io_context from draining.
+    std::vector<std::shared_ptr<http_session>> stopping;
+    for (auto* lists : { &idle_sessions, &busy_sessions, &pending_sessions }) {
+      for (auto& [type, sessions] : *lists) {
+        for (auto& s : sessions) {
+          if (s) {
+            stopping.emplace_back(std::move(s));
+          }
         }
       }
     }
-    for (auto& [type, sessions] : busy_sessions) {
-      for (auto& s : sessions) {
-        if (s) {
-          s->stop();
-        }
+    if (stopping.empty()) {
+      if (on_stopped) {
+        on_stopped();
       }
+      return;
     }
-    for (auto& [type, sessions] : pending_sessions) {
-      for (auto& s : sessions) {
-        if (s) {
+    auto remaining = std::make_shared<std::atomic_size_t>(stopping.size());
+    auto done = std::make_shared<utils::movable_function<void()>>(std::move(on_stopped));
+    for (auto& s : stopping) {
+      asio::post(s->get_executor(), [s, remaining, done]() {
+        // A stop() that throws has still torn the session down, so its reads still drain.
+        std::exception_ptr error{};
+        try {
           s->stop();
+        } catch (...) {
+          error = std::current_exception();
         }
-      }
+        s->on_reads_drained([remaining, done]() {
+          if (remaining->fetch_sub(1) == 1 && *done) {
+            (*done)();
+          }
+        });
+        if (error) {
+          std::rethrow_exception(error);
+        }
+      });
     }
   }
 
   template<typename Request, typename Handler>
   void execute(Request request, Handler&& handler)
   {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    if (!configured_) {
-      return defer_command(request, std::move(handler));
-    }
-#endif
     std::string preferred_node;
     if constexpr (http_traits::supports_sticky_node_v<Request>) {
       if (request.send_to_node) {
@@ -513,16 +526,6 @@ public:
       return handler(request.make_response(std::move(ctx), encoded_response_type{}));
     }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    auto cmd = std::make_shared<operations::http_command<Request>>(
-      ctx_,
-      request,
-      tracer_,
-      meter_,
-      app_telemetry_meter_,
-      options_.default_timeout_for(request.type),
-      dispatch_timeout_);
-#else
     auto cmd = std::make_shared<operations::http_command<Request>>(
       ctx_,
       request,
@@ -530,15 +533,15 @@ public:
       meter_,
       app_telemetry_meter_,
       options_.default_timeout_for(request.type));
-#endif
 
     using response_type = typename Request::response_type;
+    // Before start(), so a deadline that fires at once still reports the session's endpoint.
+    cmd->set_command_session(session);
     cmd->start([self = shared_from_this(), cmd, handler = std::forward<Handler>(handler)](
                  response_type&& resp) mutable {
       handler(std::move(resp));
-      self->check_in(cmd->request.type, cmd->session_);
+      self->check_in(cmd->request.type, cmd->session_for_check_in());
     });
-    cmd->set_command_session(session);
     if (!session->is_connected()) {
       connect_then_send(session, cmd, preferred_node);
     } else {
@@ -546,49 +549,34 @@ public:
     }
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
   void connect_then_send_pending_op(
     std::shared_ptr<http_session> session,
     const std::string& preferred_node,
-    std::chrono::time_point<std::chrono::steady_clock> dispatch_deadline,
     std::chrono::time_point<std::chrono::steady_clock> deadline,
     utils::movable_function<void(std::error_code, std::shared_ptr<http_session>)> callback)
-#else
-  void connect_then_send_pending_op(
-    std::shared_ptr<http_session> session,
-    const std::string& preferred_node,
-    std::chrono::time_point<std::chrono::steady_clock> deadline,
-    utils::movable_function<void(std::error_code, std::shared_ptr<http_session>)> callback)
-#endif
   {
     session->connect([self = shared_from_this(),
                       session,
                       preferred_node,
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-                      dispatch_deadline,
-#endif
                       deadline,
                       cb = std::move(callback)]() mutable {
+      // stop() runs this callback, from close() among others. A stopped session is terminal: a
+      // replacement would open a socket for an operation the stop has ended.
+      if (session->is_stopped()) {
+        return cb(errc::common::request_canceled, {});
+      }
+      // A session from before a close() gets no replacement and no reconnect: close() may not reach
+      // it, so it is stopped here.
+      if (self->is_retired(*session)) {
+        session->stop();
+        return cb(errc::common::request_canceled, {});
+      }
       if (!session->is_connected()) {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-        auto now = std::chrono::steady_clock::now();
-        if (session->is_stopped()) {
-          // Session was forced to stop (e.g. due to cluster being closed or cancellation)
-          cb(errc::common::request_canceled, {});
-          return;
-        }
-        if (dispatch_deadline < now || deadline < now) {
-          session->stop();
-          cb(errc::common::unambiguous_timeout, {});
-          return;
-        }
-#else
         if (deadline < std::chrono::steady_clock::now()) {
           session->stop();
           return cb(errc::common::unambiguous_timeout, {});
           return;
         }
-#endif
 
         // stop this session and create a new one w/ new hostname + port
         session->stop();
@@ -599,76 +587,34 @@ public:
           cb(errc::common::service_not_available, {});
           return;
         }
-        auto new_session = self->create_session(session->type(), node);
+        auto new_session = self->create_replacement(session->type(), node, *session);
+        if (!new_session) {
+          return cb(errc::common::request_canceled, {});
+        }
         if (new_session->is_connected()) {
-          {
-            const std::scoped_lock inner_lock(self->sessions_mutex_);
-            self->busy_sessions_[new_session->type()].push_back(new_session);
+          if (!self->publish_busy(new_session, /* leaving_pending */ true)) {
+            new_session->stop();
+            return cb(errc::common::request_canceled, {});
           }
           cb({}, new_session);
         } else {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-          {
-            const std::scoped_lock inner_lock(self->sessions_mutex_);
-            self->pending_sessions_[new_session->type()].push_back(new_session);
-          }
-          self->connect_then_send_pending_op(
-            new_session, preferred_node, dispatch_deadline, deadline, cb);
-#else
-          self->connect_then_send_pending_op(new_session, preferred_node, deadline, cb);
-#endif
+          self->connect_then_send_pending_op(new_session, preferred_node, deadline, std::move(cb));
         }
       } else {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-        auto now = std::chrono::steady_clock::now();
-        if (dispatch_deadline < now || deadline < now) {
-          session->stop();
-          cb(errc::common::unambiguous_timeout, {});
-          return;
-        }
-#else
         if (deadline < std::chrono::steady_clock::now()) {
           session->stop();
           cb(errc::common::unambiguous_timeout, {});
           return;
         }
-#endif
-        {
-          const std::scoped_lock inner_lock(self->sessions_mutex_);
-          self->busy_sessions_[session->type()].push_back(session);
-          self->pending_sessions_[session->type()].remove_if(
-            [id = session->id()](const auto& s) -> bool {
-              return !s || s->id() == id;
-            });
+        if (!self->publish_busy(session, /* leaving_pending */ true)) {
+          session->stop();
+          return cb(errc::common::request_canceled, {});
         }
         cb({}, session);
       }
     });
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  void add_to_deferred_queue(utils::movable_function<void(error_union)> command)
-  {
-    const std::scoped_lock lock_for_deferred_commands(deferred_commands_mutex_);
-    deferred_commands_.emplace(std::move(command));
-  }
-
-  [[nodiscard]] auto dispatch_timeout() const -> std::chrono::milliseconds
-  {
-    return dispatch_timeout_;
-  }
-
-  [[nodiscard]] auto is_configured() const -> bool
-  {
-    return configured_;
-  }
-
-  [[nodiscard]] auto last_bootstrap_error() const -> std::optional<impl::bootstrap_error>
-  {
-    return last_bootstrap_error_;
-  }
-#endif
-
 private:
   template<typename Request>
   void connect_then_send(std::shared_ptr<http_session> session,
@@ -681,19 +627,27 @@ private:
                       cmd,
                       preferred_node = std::move(preferred_node),
                       reuse_session]() mutable {
+      // stop() runs this callback, from close() among others. A stopped session is terminal: a
+      // replacement would open a socket and send a request the stop has ended, and with
+      // reuse_session the stopped session's connect() would run this callback again inline, an
+      // unbounded recursion.
+      if (session->is_stopped()) {
+        return cmd->invoke_handler(errc::common::request_canceled, {});
+      }
+      // A session from before a close() gets no replacement and no reconnect: close() may not reach
+      // it, so it is stopped here.
+      if (self->is_retired(*session)) {
+        session->stop();
+        return cmd->invoke_handler(errc::common::request_canceled, {});
+      }
       if (!session->is_connected()) {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-        auto now = std::chrono::steady_clock::now();
-        if (cmd->dispatch_deadline_expiry() < now || cmd->deadline_expiry() < now) {
-          // The http command will stop its session when the deadline expires.
-          return;
-        }
-#else
         if (cmd->deadline_expiry() < std::chrono::steady_clock::now()) {
-          // The http command will stop its session when the deadline expires.
+          // The command's deadline completes it, but stops only the session it held when it
+          // fired, which may be the one this replaced. Stopped here, a replacement does not stay
+          // listed pending until close().
+          session->stop();
           return;
         }
-#endif
         if (reuse_session) {
           return self->connect_then_send(session, cmd, preferred_node, reuse_session);
         }
@@ -706,25 +660,97 @@ private:
           cmd->invoke_handler(errc::common::service_not_available, {});
           return;
         }
-        auto new_session = self->create_session(session->type(), node);
+        auto new_session = self->create_replacement(session->type(), node, *session);
+        if (!new_session) {
+          return cmd->invoke_handler(errc::common::request_canceled, {});
+        }
         cmd->set_command_session(new_session);
         if (new_session->is_connected()) {
-          std::scoped_lock inner_lock(self->sessions_mutex_);
-          self->busy_sessions_[new_session->type()].push_back(new_session);
+          if (!self->publish_busy(new_session, /* leaving_pending */ true)) {
+            new_session->stop();
+            return cmd->invoke_handler(errc::common::request_canceled, {});
+          }
           cmd->send_to();
         } else {
           self->connect_then_send(new_session, cmd, preferred_node);
         }
       } else {
-        std::scoped_lock inner_lock(self->sessions_mutex_);
-        self->busy_sessions_[session->type()].push_back(session);
+        if (!self->publish_busy(session, /* leaving_pending */ true)) {
+          session->stop();
+          return cmd->invoke_handler(errc::common::request_canceled, {});
+        }
+        // Outside sessions_mutex_: send_to() can complete the command inline (an encode_to
+        // failure, or a session already stopped), and the completion calls check_in, which takes
+        // it.
         cmd->send_to();
       }
     });
   }
 
-  auto create_session(service_type type, const node_details& node) -> std::shared_ptr<http_session>
+  // Lists a connected session as busy unless its stop() has begun or it is from before a close(),
+  // and reports whether it did. stop() sets stopped_ before on_stop's cleanup takes
+  // sessions_mutex_, so a stop is either seen here or removes the session after it is listed.
+  // Tested outside the lock, a stop landing in between would list a session whose cleanup has
+  // already run. A refused session is the caller's to stop: close() does not reach one it never had
+  // in a list.
+  auto publish_busy(const std::shared_ptr<http_session>& session, bool leaving_pending = false)
+    -> bool
+  {
+    const std::scoped_lock lock(sessions_mutex_);
+    if (session->pool_generation() != generation_ || session->is_stopped()) {
+      return false;
+    }
+    busy_sessions_[session->type()].push_back(session);
+    if (leaving_pending) {
+      pending_sessions_[session->type()].remove_if([id = session->id()](const auto& s) -> bool {
+        return !s || s->id() == id;
+      });
+    }
+    return true;
+  }
+
+  auto current_generation() -> std::uint64_t
+  {
+    const std::scoped_lock lock(sessions_mutex_);
+    return generation_;
+  }
+
+  // Serves a generation that a close() has ended.
+  auto is_retired(const http_session& session) -> bool
+  {
+    return session.pool_generation() != current_generation();
+  }
+
+  // A popped idle session goes back into service only if nothing has claimed it and reset_idle()
+  // accepts it, which it refuses once the idle timer has fired or a stop() has begun.
+  static auto is_reusable(http_session& session) -> bool
+  {
+    return !session.is_stopping() && session.reset_idle();
+  }
+
+  // A failover replacement for `failed`. It serves the same operation, so it belongs to the same
+  // generation, and it is listed pending from creation: close() either stops it or has already
+  // retired that generation, and then the replacement is discarded and nullptr returned. Unlisted
+  // until its connect completes, it would be invisible to a close() whose completion lets the
+  // io_context stop.
+  auto create_replacement(service_type type, const node_details& node, const http_session& failed)
+    -> std::shared_ptr<http_session>
+  {
+    auto session = create_session(type, node, failed.pool_generation());
+    const std::scoped_lock lock(sessions_mutex_);
+    if (session->pool_generation() != generation_) {
+      return nullptr;
+    }
+    pending_sessions_[type].push_back(session);
+    return session;
+  }
+
+  auto create_session(service_type type, const node_details& node, std::uint64_t generation)
+    -> std::shared_ptr<http_session>
   {
+    // Read before the session takes the TLS context. retire_sessions() runs after the context is
+    // replaced, so this epoch is never newer than the context.
+    const std::uint64_t tls_epoch = tls_epoch_;
     std::shared_ptr<http_session> session;
     if (options_.enable_tls) {
       session = std::make_shared<http_session>(type,
@@ -743,7 +769,8 @@ private:
                                                  node.port,
                                                  node.canonical_hostname,
                                                  node.canonical_port,
-                                               });
+                                               },
+                                               generation);
     } else {
       session = std::make_shared<http_session>(type,
                                                client_id_,
@@ -760,7 +787,8 @@ private:
                                                  node.port,
                                                  node.canonical_hostname,
                                                  node.canonical_port,
-                                               });
+                                               },
+                                               generation);
     }
 
     session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {
@@ -789,87 +817,10 @@ private:
         }
       }
     });
+    session->set_tls_epoch(tls_epoch);
     return session;
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  template<typename Request, typename Handler>
-  void defer_command(Request request, Handler&& handler)
-  {
-    {
-      std::scoped_lock bootstrap_error_lock(last_bootstrap_error_mutex_);
-      if (last_bootstrap_error_.has_value()) {
-        typename Request::error_context_type ctx{};
-        ctx.ec = last_bootstrap_error_.value().ec;
-        using response_type = typename Request::encoded_response_type;
-        return handler(request.make_response(std::move(ctx), response_type{}));
-      }
-    }
-    auto cmd = std::make_shared<operations::http_command<Request>>(
-      ctx_,
-      request,
-      tracer_,
-      meter_,
-      app_telemetry_meter_,
-      options_.default_timeout_for(request.type),
-      dispatch_timeout_);
-    using response_type = typename Request::response_type;
-    cmd->start([self = shared_from_this(), cmd, handler = std::forward<Handler>(handler)](
-                 response_type&& resp) mutable {
-      handler(std::move(resp));
-      self->check_in(cmd->request.type, cmd->session_);
-    });
-    CB_LOG_DEBUG(R"(Adding HTTP request to deferred queue: {}, client_context_id="{}")",
-                 cmd->request.type,
-                 cmd->client_context_id_);
-    add_to_deferred_queue([self = shared_from_this(), cmd, request](error_union err) mutable {
-      if (!std::holds_alternative<std::monostate>(err)) {
-        using response_type = typename Request::encoded_response_type;
-        return cmd->invoke_handler(err, response_type{});
-      }
-
-      // don't do anything if the command wasn't dispatched or has already timed out
-      auto now = std::chrono::steady_clock::now();
-      if (cmd->dispatch_deadline_expiry() < now || cmd->deadline_expiry() < now) {
-        return;
-      }
-      std::string preferred_node;
-      if constexpr (http_traits::supports_sticky_node_v<Request>) {
-        if (request.send_to_node) {
-          preferred_node = *request.send_to_node;
-        }
-      }
-      auto [error, session] = self->check_out(request.type, preferred_node);
-      if (error) {
-        using response_type = typename Request::encoded_response_type;
-        return cmd->invoke_handler(error, response_type{});
-      }
-      cmd->set_command_session(session);
-      if (!session->is_connected()) {
-        self->connect_then_send(session, cmd, preferred_node);
-      } else {
-        cmd->send_to();
-      }
-    });
-  }
-
-  void drain_deferred_queue(error_union err)
-  {
-    std::queue<utils::movable_function<void(error_union)>> commands{};
-    {
-      const std::scoped_lock lock(deferred_commands_mutex_);
-      std::swap(deferred_commands_, commands);
-    }
-    if (!commands.empty()) {
-      CB_LOG_TRACE("Draining deferred operation queue, size={}", commands.size());
-    }
-    while (!commands.empty()) {
-      commands.front()(err);
-      commands.pop();
-    }
-  }
-#endif
-
   auto next_node(service_type type) -> node_details
   {
     std::scoped_lock lock(config_mutex_);
@@ -968,15 +919,11 @@ private:
   std::size_t next_index_{ 0 };
   std::mutex next_index_mutex_{};
   std::mutex sessions_mutex_{};
+  // Advanced by close(); guarded by sessions_mutex_. A session carries the generation it serves,
+  // and check_in and publish_busy refuse one from an earlier generation.
+  std::uint64_t generation_{ 0 };
+  // Advanced by retire_sessions(). Unlike generation_, it ends no in-flight request.
+  std::atomic<std::uint64_t> tls_epoch_{ 0 };
   query_cache query_cache_{};
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  std::atomic_bool configured_{ false };
-  std::chrono::milliseconds dispatch_timeout_{};
-  std::atomic_bool allow_fast_fail_{ true };
-  std::queue<utils::movable_function<void(error_union)>> deferred_commands_{};
-  std::mutex deferred_commands_mutex_{};
-  std::optional<impl::bootstrap_error> last_bootstrap_error_;
-  std::mutex last_bootstrap_error_mutex_{};
-#endif
 };
 } // namespace couchbase::core::io