couchbase 3.8.2 → 3.9.1

data/ext/couchbase/core/http_component.cxx

@@ -19,10 +19,11 @@
 
 #include "free_form_http_request.hxx"
 #include "io/http_session_manager.hxx"
+#include "logger/redaction.hxx"
 #include "pending_operation.hxx"
 #include "pending_operation_connection_info.hxx"
-
-#include <couchbase/build_config.hxx>
+#include "tracing/constants.hxx"
+#include "tracing/tracer_wrapper.hxx"
 
 #include <asio/error.hpp>
 #include <spdlog/fmt/bundled/chrono.h>
@@ -50,25 +51,12 @@ class pending_http_operation
   , public pending_operation_connection_info
 {
 public:
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  pending_http_operation(asio::io_context& io,
-                         http_request request,
-                         std::chrono::milliseconds dispatch_timeout)
-    : deadline_{ io }
-    , dispatch_deadline_{ io }
-    , dispatch_timeout_{ dispatch_timeout }
-    , request_{ std::move(request) }
-    , encoded_{ encode_http_request(request_) }
-  {
-  }
-#else
   pending_http_operation(asio::io_context& io, http_request request)
     : deadline_{ io }
     , request_{ std::move(request) }
     , encoded_{ encode_http_request(request_) }
   {
   }
-#endif
 
   ~pending_http_operation() override = default;
   pending_http_operation(const pending_http_operation&) = delete;
@@ -80,107 +68,112 @@ public:
   {
     callback_ = std::move(callback);
     encoded_.headers["client-context-id"] = request_.client_context_id;
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    dispatch_deadline_.expires_after(dispatch_timeout_);
-    dispatch_deadline_.async_wait([self = shared_from_this()](auto ec) {
-      if (ec == asio::error::operation_aborted) {
-        return;
-      }
-      CB_LOG_DEBUG(
-        R"(HTTP request timed out (dispatch): {}, method={}, path="{}", dispatch_timeout={}, client_context_id={})",
-        self->encoded_.type,
-        self->encoded_.method,
-        self->encoded_.path,
-        self->dispatch_timeout_,
-        self->encoded_.client_context_id);
-      self->trigger_timeout();
-      if (self->session_) {
-        self->session_->stop();
-      }
-    });
-#endif
     deadline_.expires_after(request_.timeout);
     deadline_.async_wait([self = shared_from_this()](auto ec) {
       if (ec == asio::error::operation_aborted) {
         return;
       }
+      // deadline_.cancel() does not retract a completion already queued. A response that took
+      // the callback first has handed the session to the stream or to the pool.
+      if (!self->trigger_timeout()) {
+        return;
+      }
       CB_LOG_DEBUG(
         R"(HTTP request timed out: {}, method={}, path="{}", timeout={}, client_context_id={})",
         self->encoded_.type,
         self->encoded_.method,
-        self->encoded_.path,
+        logger::user_data(self->encoded_.path),
         self->request_.timeout,
         self->encoded_.client_context_id);
-      self->trigger_timeout();
-      if (self->session_) {
-        self->session_->stop();
-      }
     });
   }
 
-  void set_stream_end_callback(utils::movable_function<void()>&& stream_end_callback)
+  // Receives the session the stream was dispatched on, which a failover makes different from the
+  // one checked out.
+  void set_stream_end_callback(
+    utils::movable_function<void(std::shared_ptr<io::http_session>)>&& stream_end_callback)
   {
     stream_end_callback_ = std::move(stream_end_callback);
   }
 
+  void set_tracer(std::shared_ptr<tracing::tracer_wrapper> tracer)
+  {
+    tracer_ = std::move(tracer);
+  }
+
+  // Once the stream has ended its connection is back in the pool, possibly serving another
+  // request, so cancel() leaves it alone. A cancel that comes first keeps the stream end from
+  // checking the connection in.
   void cancel() override
   {
-    if (session_) {
-      session_->stop();
+    std::shared_ptr<io::http_session> session;
+    {
+      const std::scoped_lock lock(callback_mutex_);
+      if (stream_ended_) {
+        return;
+      }
+      cancelled_ = true;
+      session = session_.lock();
+    }
+    if (session) {
+      session->stop();
     }
     invoke_response_handler(errc::common::request_canceled, {});
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  void invoke_response_handler(error_union err, io::http_streaming_response resp)
+  // Returns whether this call took the callback, which is the token for completing the request.
+  auto invoke_response_handler(std::error_code err, io::http_streaming_response resp) -> bool
   {
-    dispatch_deadline_.cancel();
-#else
-  void invoke_response_handler(std::error_code err, io::http_streaming_response resp)
-  {
-#endif
     deadline_.cancel();
     free_form_http_request_callback callback{};
+    std::shared_ptr<couchbase::tracing::request_span> dispatch_span{};
     {
       const std::scoped_lock lock(callback_mutex_);
       std::swap(callback, callback_);
+      std::swap(dispatch_span, dispatch_span_);
+    }
+    // The dispatch span covers the request up to the response headers, not the streamed body.
+    if (dispatch_span) {
+      dispatch_span->end();
     }
-    if (callback) {
-      callback(http_response{ std::move(resp) }, err);
+    if (!callback) {
+      return false;
     }
+    callback(http_response{ std::move(resp) }, err);
+    return true;
   }
 
   void send_to(std::shared_ptr<io::http_session> session)
   {
-    if (!callback_) {
-      return;
+    {
+      // The deadline or a cancel can complete the operation concurrently. Checking the callback and
+      // storing the span under the lock that invoke_response_handler takes means the span is either
+      // stored before the handler runs, and ended by it, or never created.
+      std::unique_lock lock(callback_mutex_);
+      if (!callback_) {
+        // The session is listed busy and nothing checks it in; on_stop removes it.
+        lock.unlock();
+        return session->stop();
+      }
+      session_ = session;
+      dispatched_to_ = session->remote_address();
+      dispatched_from_ = session->local_address();
+      dispatched_to_host_ = fmt::format("{}:{}", session->hostname(), session->port());
+      dispatch_span_ = create_dispatch_span(*session);
     }
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    dispatch_deadline_.cancel();
-#endif
-    session_ = std::move(session);
 
-    auto start_op = [self = shared_from_this()]() {
-      self->session_->write_and_stream(
+    auto start_op = [self = shared_from_this(), session]() {
+      session->write_and_stream(
         self->encoded_,
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-        [self](error_union err, io::http_streaming_response resp) {
-          if (std::holds_alternative<std::error_code>(err) &&
-              std::get<std::error_code>(err) == asio::error::operation_aborted) {
-            return;
-          }
-          self->invoke_response_handler(err, std::move(resp));
-        },
-#else
         [self](std::error_code ec, io::http_streaming_response resp) {
           if (ec == asio::error::operation_aborted) {
             return;
           }
+          self->refresh_dispatched();
           self->invoke_response_handler(ec, std::move(resp));
         },
-#endif
         [self]() {
-          self->stream_end_callback_();
+          self->stream_end_callback_(self->session_at_stream_end());
         });
     };
 
@@ -192,14 +185,6 @@ public:
     return deadline_.expiry();
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  [[nodiscard]] auto dispatch_deadline_expiry() const
-    -> std::chrono::time_point<std::chrono::steady_clock>
-  {
-    return dispatch_deadline_.expiry();
-  }
-#endif
-
   [[nodiscard]] auto request() const -> http_request
   {
     return request_;
@@ -207,40 +192,137 @@ public:
 
   [[nodiscard]] auto dispatched_to() const -> std::string override
   {
-    return session_->remote_address();
+    const std::scoped_lock lock(callback_mutex_);
+    return dispatched_to_;
   }
 
   [[nodiscard]] auto dispatched_from() const -> std::string override
   {
-    return session_->local_address();
+    const std::scoped_lock lock(callback_mutex_);
+    return dispatched_from_;
   }
 
   [[nodiscard]] auto dispatched_to_host() const -> std::string override
   {
-    return fmt::format("{}:{}", session_->hostname(), session_->port());
+    const std::scoped_lock lock(callback_mutex_);
+    return dispatched_to_host_;
   }
 
 private:
-  void trigger_timeout()
+  // Rereads the endpoints from the session. A resend after a refused client certificate writes the
+  // request on a new connection.
+  void refresh_dispatched()
+  {
+    const std::scoped_lock lock(callback_mutex_);
+    if (const auto session = session_.lock(); session) {
+      if (auto [from, to] = session->dispatched_endpoints(); !to.empty()) {
+        dispatched_from_ = std::move(from);
+        dispatched_to_ = std::move(to);
+      }
+    }
+  }
+
+  // Null before send_to() runs or once the session is gone: the operation does not keep its
+  // session alive.
+  [[nodiscard]] auto session() const -> std::shared_ptr<io::http_session>
+  {
+    const std::scoped_lock lock(callback_mutex_);
+    return session_.lock();
+  }
+
+  // The session to check in at stream end: null if cancel() or the deadline came first, which stops
+  // it.
+  [[nodiscard]] auto session_at_stream_end() -> std::shared_ptr<io::http_session>
   {
+    const std::scoped_lock lock(callback_mutex_);
+    stream_ended_ = true;
+    if (cancelled_) {
+      return nullptr;
+    }
+    return session_.lock();
+  }
+
+  // Ends the request with a timeout unless a response took the callback first. The session is
+  // stopped before the callback runs, and a stream end does not check it in. Returns whether this
+  // call took the callback.
+  auto trigger_timeout() -> bool
+  {
+    refresh_dispatched();
     // TODO(JC):  if triggered from the dispatch timeout, should only be
     // errc::common::unambiguous_timeout?
     auto ec =
       request_.is_read_only ? errc::common::unambiguous_timeout : errc::common::ambiguous_timeout;
-    invoke_response_handler(ec, {});
+    free_form_http_request_callback callback{};
+    std::shared_ptr<couchbase::tracing::request_span> dispatch_span{};
+    std::shared_ptr<io::http_session> session;
+    {
+      const std::scoped_lock lock(callback_mutex_);
+      std::swap(callback, callback_);
+      std::swap(dispatch_span, dispatch_span_);
+      if (callback) {
+        cancelled_ = true;
+      }
+      session = session_.lock();
+    }
+    if (dispatch_span) {
+      dispatch_span->end();
+    }
+    if (!callback) {
+      return false;
+    }
+    if (session) {
+      session->stop();
+    }
+    callback(http_response{ io::http_streaming_response{} }, ec);
+    return true;
+  }
+
+  // Carries the same tags as http_command::create_dispatch_span, so a streamed request reports the
+  // same dispatch_to_server span as a buffered one. It is parented to the caller's span because no
+  // operation span exists in core for a streamed request.
+  [[nodiscard]] auto create_dispatch_span(io::http_session& session) const
+    -> std::shared_ptr<couchbase::tracing::request_span>
+  {
+    if (!tracer_) {
+      return {};
+    }
+    auto dispatch_span =
+      tracer_->create_span(tracing::operation::step_dispatch, request_.parent_span);
+    if (dispatch_span->uses_tags()) {
+      dispatch_span->add_tag(tracing::attributes::dispatch::network_transport, "tcp");
+      dispatch_span->add_tag(tracing::attributes::dispatch::operation_id,
+                             request_.client_context_id);
+      dispatch_span->add_tag(tracing::attributes::dispatch::local_id, session.id());
+      dispatch_span->add_tag(tracing::attributes::dispatch::server_address,
+                             session.http_context().canonical_hostname);
+      dispatch_span->add_tag(tracing::attributes::dispatch::server_port,
+                             session.http_context().canonical_port);
+
+      const auto& peer_endpoint = session.remote_endpoint();
+      dispatch_span->add_tag(tracing::attributes::dispatch::peer_address,
+                             peer_endpoint.address().to_string());
+      dispatch_span->add_tag(tracing::attributes::dispatch::peer_port, peer_endpoint.port());
+    }
+    return dispatch_span;
   }
 
   asio::steady_timer deadline_;
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  asio::steady_timer dispatch_deadline_;
-  std::chrono::milliseconds dispatch_timeout_;
-#endif
   http_request request_;
   io::http_request encoded_;
   free_form_http_request_callback callback_;
-  utils::movable_function<void()> stream_end_callback_;
-  std::shared_ptr<io::http_session> session_;
-  std::mutex callback_mutex_;
+  std::shared_ptr<tracing::tracer_wrapper> tracer_{};
+  std::shared_ptr<couchbase::tracing::request_span> dispatch_span_{};
+  utils::movable_function<void(std::shared_ptr<io::http_session>)> stream_end_callback_;
+  std::weak_ptr<io::http_session> session_;
+  // Endpoints of the session send_to() dispatched on, guarded by callback_mutex_. Recorded because
+  // session_ expires once the session is released.
+  std::string dispatched_to_;
+  std::string dispatched_from_;
+  std::string dispatched_to_host_;
+  // Guarded by callback_mutex_.
+  bool stream_ended_{ false };
+  bool cancelled_{ false };
+  mutable std::mutex callback_mutex_;
 };
 
 class pending_buffered_http_operation
@@ -249,25 +331,12 @@ class pending_buffered_http_operation
   , public pending_operation_connection_info
 {
 public:
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  pending_buffered_http_operation(asio::io_context& io,
-                                  http_request request,
-                                  std::chrono::milliseconds dispatch_timeout)
-    : deadline_{ io }
-    , dispatch_deadline_{ io }
-    , dispatch_timeout_{ dispatch_timeout }
-    , request_{ std::move(request) }
-    , encoded_{ encode_http_request(request_) }
-  {
-  }
-#else
   pending_buffered_http_operation(asio::io_context& io, http_request request)
     : deadline_{ io }
     , request_{ std::move(request) }
     , encoded_{ encode_http_request(request_) }
   {
   }
-#endif
 
   ~pending_buffered_http_operation() override = default;
   pending_buffered_http_operation(const pending_buffered_http_operation&) = delete;
@@ -280,99 +349,106 @@ public:
   {
     callback_ = std::move(callback);
     encoded_.headers["client-context-id"] = request_.client_context_id;
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    dispatch_deadline_.expires_after(dispatch_timeout_);
-    dispatch_deadline_.async_wait([self = shared_from_this()](auto ec) {
-      if (ec == asio::error::operation_aborted) {
-        return;
-      }
-      CB_LOG_DEBUG(
-        R"(HTTP request timed out (dispatch): {}, method={}, path="{}", dispatch_timeout={}, client_context_id={})",
-        self->encoded_.type,
-        self->encoded_.method,
-        self->encoded_.path,
-        self->dispatch_timeout_,
-        self->encoded_.client_context_id);
-      self->trigger_timeout();
-      if (self->session_) {
-        self->session_->stop();
-      }
-    });
-#endif
     deadline_.expires_after(request_.timeout);
     deadline_.async_wait([self = shared_from_this()](auto ec) {
       if (ec == asio::error::operation_aborted) {
         return;
       }
+      // deadline_.cancel() does not retract a completion already queued. A response that took
+      // the callback first has handed the session back to the pool.
+      if (!self->trigger_timeout()) {
+        return;
+      }
       CB_LOG_DEBUG(
         R"(HTTP request timed out: {}, method={}, path="{}", timeout={}, client_context_id={})",
         self->encoded_.type,
         self->encoded_.method,
-        self->encoded_.path,
+        logger::user_data(self->encoded_.path),
         self->request_.timeout,
         self->encoded_.client_context_id);
-      self->trigger_timeout();
-      if (self->session_) {
-        self->session_->stop();
-      }
     });
   }
 
+  // Takes the callback, then stops the session before running it: the callback checks the session
+  // in, and check_in refuses a stopped one. With no callback left the operation has completed, and
+  // its connection may be serving another request.
   void cancel() override
   {
-    if (session_) {
-      session_->stop();
+    buffered_free_form_http_request_callback callback{};
+    std::shared_ptr<io::http_session> session;
+    {
+      const std::scoped_lock lock(callback_mutex_);
+      std::swap(callback, callback_);
+      session = session_.lock();
     }
-    invoke_response_handler(errc::common::request_canceled, {});
+    if (!callback) {
+      return;
+    }
+    deadline_.cancel();
+    if (session) {
+      session->stop();
+    }
+    callback(buffered_http_response{ io::http_response{} }, errc::common::request_canceled);
   }
 
-  void invoke_response_handler(std::error_code ec, io::http_response resp)
+  // Returns whether this call took the callback, which is the token for completing the request.
+  auto invoke_response_handler(std::error_code ec, io::http_response resp) -> bool
   {
     deadline_.cancel();
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    dispatch_deadline_.cancel();
-#endif
     buffered_free_form_http_request_callback callback{};
     {
       const std::scoped_lock lock(callback_mutex_);
       std::swap(callback, callback_);
     }
-    if (callback) {
-      callback(buffered_http_response{ std::move(resp) }, ec);
+    if (!callback) {
+      return false;
     }
+    callback(buffered_http_response{ std::move(resp) }, ec);
+    return true;
   }
 
   void send_to(std::shared_ptr<io::http_session> session)
   {
-    if (!callback_) {
-      return;
+    {
+      // Checked and stored under the lock invoke_response_handler takes the callback with: a
+      // deadline completion that took the callback reads this session, and after it took the
+      // callback nothing is stored.
+      std::unique_lock lock(callback_mutex_);
+      if (!callback_) {
+        // The completion checks in only a session stored here, so this one stays listed busy
+        // until on_stop removes it.
+        lock.unlock();
+        return session->stop();
+      }
+      session_ = session;
+      dispatched_to_ = session->remote_address();
+      dispatched_from_ = session->local_address();
+      dispatched_to_host_ = fmt::format("{}:{}", session->hostname(), session->port());
     }
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    dispatch_deadline_.cancel();
-#endif
-    session_ = std::move(session);
 
-    session_->write_and_subscribe(
+    session->write_and_subscribe(
       encoded_, [self = shared_from_this()](std::error_code ec, io::http_response resp) {
         if (ec == asio::error::operation_aborted) {
           return;
         }
+        self->refresh_dispatched();
         self->invoke_response_handler(ec, std::move(resp));
       });
   }
 
-  [[nodiscard]] auto deadline_expiry() const -> std::chrono::time_point<std::chrono::steady_clock>
+  // The session send_to() dispatched on, which a failover makes different from the one checked
+  // out. Null if send_to() never ran or the session is gone: the operation does not keep its
+  // session alive.
+  [[nodiscard]] auto session() const -> std::shared_ptr<io::http_session>
   {
-    return deadline_.expiry();
+    const std::scoped_lock lock(callback_mutex_);
+    return session_.lock();
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  [[nodiscard]] auto dispatch_deadline_expiry() const
-    -> std::chrono::time_point<std::chrono::steady_clock>
+  [[nodiscard]] auto deadline_expiry() const -> std::chrono::time_point<std::chrono::steady_clock>
   {
-    return dispatch_deadline_.expiry();
+    return deadline_.expiry();
   }
-#endif
 
   [[nodiscard]] auto request() const -> http_request
   {
@@ -381,39 +457,74 @@ public:
 
   [[nodiscard]] auto dispatched_to() const -> std::string override
   {
-    return session_->remote_address();
+    const std::scoped_lock lock(callback_mutex_);
+    return dispatched_to_;
   }
 
   [[nodiscard]] auto dispatched_from() const -> std::string override
   {
-    return session_->local_address();
+    const std::scoped_lock lock(callback_mutex_);
+    return dispatched_from_;
   }
 
   [[nodiscard]] auto dispatched_to_host() const -> std::string override
   {
-    return fmt::format("{}:{}", session_->hostname(), session_->port());
+    const std::scoped_lock lock(callback_mutex_);
+    return dispatched_to_host_;
   }
 
 private:
-  void trigger_timeout()
+  // Rereads the endpoints from the session. A resend after a refused client certificate writes the
+  // request on a new connection.
+  void refresh_dispatched()
+  {
+    const std::scoped_lock lock(callback_mutex_);
+    if (const auto session = session_.lock(); session) {
+      if (auto [from, to] = session->dispatched_endpoints(); !to.empty()) {
+        dispatched_from_ = std::move(from);
+        dispatched_to_ = std::move(to);
+      }
+    }
+  }
+
+  // Ends the request with a timeout unless a response took the callback first. The session is
+  // stopped before the callback runs: the callback checks the session in, and check_in refuses a
+  // stopped one. Returns whether this call took the callback.
+  auto trigger_timeout() -> bool
   {
+    refresh_dispatched();
     // TODO(JC):  if triggered from the dispatch timeout, should only be
     // errc::common::unambiguous_timeout?
     auto ec =
       request_.is_read_only ? errc::common::unambiguous_timeout : errc::common::ambiguous_timeout;
-    invoke_response_handler(ec, {});
+    buffered_free_form_http_request_callback callback{};
+    std::shared_ptr<io::http_session> session;
+    {
+      const std::scoped_lock lock(callback_mutex_);
+      std::swap(callback, callback_);
+      session = session_.lock();
+    }
+    if (!callback) {
+      return false;
+    }
+    if (session) {
+      session->stop();
+    }
+    callback(buffered_http_response{ io::http_response{} }, ec);
+    return true;
   }
 
   asio::steady_timer deadline_;
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  asio::steady_timer dispatch_deadline_;
-  std::chrono::milliseconds dispatch_timeout_;
-#endif
   http_request request_;
   io::http_request encoded_;
   buffered_free_form_http_request_callback callback_;
-  std::shared_ptr<io::http_session> session_;
-  std::mutex callback_mutex_;
+  std::weak_ptr<io::http_session> session_;
+  // Endpoints of the session send_to() dispatched on, guarded by callback_mutex_. Recorded because
+  // session_ expires once the session is released.
+  std::string dispatched_to_;
+  std::string dispatched_from_;
+  std::string dispatched_to_host_;
+  mutable std::mutex callback_mutex_;
 };
 
 class http_component_impl
@@ -428,13 +539,8 @@ public:
   {
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  auto do_http_request(const http_request& request, free_form_http_request_callback&& callback)
-    -> tl::expected<std::shared_ptr<pending_operation>, error_union>
-#else
   auto do_http_request(const http_request& request, free_form_http_request_callback&& callback)
     -> tl::expected<std::shared_ptr<pending_operation>, std::error_code>
-#endif
   {
     std::shared_ptr<io::http_session_manager> session_manager;
     {
@@ -444,19 +550,7 @@ public:
       }
       session_manager = std::move(sm);
     }
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    auto op =
-      std::make_shared<pending_http_operation>(io_, request, session_manager->dispatch_timeout());
-    if (!session_manager->is_configured()) {
-      auto err = defer_command(op, session_manager, std::move(callback));
-      if (!std::holds_alternative<std::monostate>(err)) {
-        return tl::unexpected{ err };
-      }
-      return op;
-    }
-#else
     auto op = std::make_shared<pending_http_operation>(io_, request);
-#endif
 
     send_http_operation(op, session_manager, std::move(callback));
     return op;
@@ -475,22 +569,7 @@ public:
       session_manager = std::move(sm);
     }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    auto op = std::make_shared<pending_buffered_http_operation>(
-      io_, request, session_manager->dispatch_timeout());
-    if (!session_manager->is_configured()) {
-      auto err = defer_command(op, session_manager, std::move(callback));
-      if (!std::holds_alternative<std::monostate>(err)) {
-        auto ec = std::holds_alternative<impl::bootstrap_error>(err)
-                    ? std::get<impl::bootstrap_error>(err).ec
-                    : std::get<std::error_code>(err);
-        return tl::unexpected{ ec };
-      }
-      return op;
-    }
-#else
     auto op = std::make_shared<pending_buffered_http_operation>(io_, request);
-#endif
 
     send_http_operation(op, session_manager, std::move(callback));
     return op;
@@ -501,44 +580,38 @@ private:
                            const std::shared_ptr<io::http_session_manager>& session_manager,
                            free_form_http_request_callback&& callback)
   {
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-    op->start([callback = std::move(callback)](auto resp, error_union err) mutable {
-      callback(std::move(resp), err);
-    });
-    // don't do anything if the op wasn't dispatched or has already timed out
-    auto now = std::chrono::steady_clock::now();
-    if (op->dispatch_deadline_expiry() < now || op->deadline_expiry() < now) {
-      return;
-    }
-#else
     op->start([callback = std::move(callback)](auto resp, auto ec) mutable {
       callback(std::move(resp), ec);
     });
-#endif
     std::shared_ptr<io::http_session> session;
     {
       auto [check_out_ec, s] = session_manager->check_out(
         op->request().service, op->request().endpoint, op->request().internal.undesired_endpoint);
       if (check_out_ec) {
-        return op->invoke_response_handler(check_out_ec, {});
+        op->invoke_response_handler(check_out_ec, {});
+        return;
       }
       session = std::move(s);
     }
+    // Not an owner: a completed operation that never reaches the stream end must not keep the
+    // manager alive.
     op->set_stream_end_callback(
-      [session_manager, session, service = op->request().service]() mutable {
-        session_manager->check_in(service, session);
+      [manager = std::weak_ptr{ session_manager },
+       service = op->request().service](std::shared_ptr<io::http_session> served) {
+        if (const auto m = manager.lock(); m) {
+          m->check_in(service, std::move(served));
+        }
       });
+    op->set_tracer(session_manager->tracer());
     if (!session->is_connected()) {
       session_manager->connect_then_send_pending_op(
         session,
         {},
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-        op->dispatch_deadline_expiry(),
-#endif
         op->deadline_expiry(),
         [op](std::error_code ec, std::shared_ptr<io::http_session> http_session) {
           if (ec) {
-            return op->invoke_response_handler(ec, {});
+            op->invoke_response_handler(ec, {});
+            return;
           }
           op->send_to(std::move(http_session));
         });
@@ -556,28 +629,33 @@ private:
       auto [check_out_ec, s] = session_manager->check_out(
         op->request().service, op->request().endpoint, op->request().internal.undesired_endpoint);
       if (check_out_ec) {
-        return op->invoke_response_handler(check_out_ec, {});
+        op->invoke_response_handler(check_out_ec, {});
+        return;
       }
       session = std::move(s);
     }
-    op->start(
-      [callback = std::move(callback), session_manager, session, service = op->request().service](
-        auto resp, auto ec) mutable {
-        callback(std::move(resp), ec);
-        session_manager->check_in(service, session);
-      });
+    // Checks in only the session send_to() stored. A completion before send_to() leaves the
+    // checked-out session to send_to(), which stops it. Checked in here, it could be taken by
+    // another request before that stop.
+    op->start([callback = std::move(callback),
+               session_manager,
+               weak_op = std::weak_ptr<pending_buffered_http_operation>{ op },
+               service = op->request().service](auto resp, auto ec) mutable {
+      callback(std::move(resp), ec);
+      if (auto self = weak_op.lock(); self) {
+        session_manager->check_in(service, self->session());
+      }
+    });
 
     if (!session->is_connected()) {
       session_manager->connect_then_send_pending_op(
         session,
         {},
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-        op->dispatch_deadline_expiry(),
-#endif
         op->deadline_expiry(),
         [op](std::error_code ec, std::shared_ptr<io::http_session> http_session) {
           if (ec) {
-            return op->invoke_response_handler(ec, {});
+            op->invoke_response_handler(ec, {});
+            return;
           }
           op->send_to(std::move(http_session));
         });
@@ -586,64 +664,6 @@ private:
     }
   }
 
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  template<typename Callback, typename PendingHttpOp>
-  auto defer_command(std::shared_ptr<PendingHttpOp> pending_op,
-                     const std::shared_ptr<io::http_session_manager>& session_manager,
-                     Callback&& callback) -> error_union
-  {
-    if (auto last_error = session_manager->last_bootstrap_error(); last_error.has_value()) {
-      return last_error.value();
-    }
-    CB_LOG_DEBUG(
-      R"(Adding pending HTTP operation to deferred queue: service={}, client_context_id={})",
-      pending_op->request().service,
-      pending_op->request().client_context_id);
-    session_manager->add_to_deferred_queue([this,
-                                            callback = std::forward<Callback>(callback),
-                                            op = std::move(pending_op),
-                                            session_manager](error_union err) mutable {
-      if (!std::holds_alternative<std::monostate>(err)) {
-        // The deferred operation was cancelled - currently this can happen due to closing the
-        // cluster
-        return callback({}, err);
-      }
-
-      return send_http_operation(op, session_manager, std::forward<Callback>(callback));
-    });
-    return std::monostate{};
-  }
-
-  auto defer_command(std::shared_ptr<pending_buffered_http_operation> pending_op,
-                     const std::shared_ptr<io::http_session_manager>& session_manager,
-                     buffered_free_form_http_request_callback&& callback) -> error_union
-  {
-    if (auto last_error = session_manager->last_bootstrap_error(); last_error.has_value()) {
-      return last_error.value();
-    }
-    CB_LOG_DEBUG(
-      R"(Adding pending HTTP operation to deferred queue: service={}, client_context_id={})",
-      pending_op->request().service,
-      pending_op->request().client_context_id);
-    session_manager->add_to_deferred_queue([this,
-                                            callback = std::move(callback),
-                                            op = std::move(pending_op),
-                                            session_manager](error_union err) mutable {
-      if (!std::holds_alternative<std::monostate>(err)) {
-        auto ec = std::holds_alternative<impl::bootstrap_error>(err)
-                    ? std::get<impl::bootstrap_error>(err).ec
-                    : std::get<std::error_code>(err);
-        // The deferred operation was cancelled - currently this can happen due to closing the
-        // cluster
-        return callback({}, ec);
-      }
-
-      return send_http_operation(op, session_manager, std::move(callback));
-    });
-    return std::monostate{};
-  }
-#endif
-
   asio::io_context& io_;
   core_sdk_shim shim_;
   std::shared_ptr<retry_strategy> default_retry_strategy_;
@@ -661,11 +681,7 @@ http_component::http_component(asio::io_context& io,
 auto
 http_component::do_http_request(const http_request& request,
                                 free_form_http_request_callback&& callback)
-#ifdef COUCHBASE_CXX_CLIENT_COLUMNAR
-  -> tl::expected<std::shared_ptr<pending_operation>, error_union>
-#else
   -> tl::expected<std::shared_ptr<pending_operation>, std::error_code>
-#endif
 {
   return impl_->do_http_request(request, std::move(callback));
 }