couchbase 3.0.0 → 3.0.1

data/ext/couchbase/io/http_session_manager.hxx

@@ -19,6 +19,7 @@
 
 #include <io/http_session.hxx>
 #include <service_type.hxx>
+#include <io/http_context.hxx>
 
 #include <random>
 
@@ -48,9 +49,27 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
         }
     }
 
+    void export_diag_info(diag::diagnostics_result& res)
+    {
+        std::scoped_lock lock(sessions_mutex_);
+
+        for (const auto& list : busy_sessions_) {
+            for (const auto& session : list.second) {
+                res.services[list.first].emplace_back(session->diag_info());
+            }
+        }
+        for (const auto& list : idle_sessions_) {
+            for (const auto& session : list.second) {
+                res.services[list.first].emplace_back(session->diag_info());
+            }
+        }
+    }
+
     std::shared_ptr<http_session> check_out(service_type type, const couchbase::cluster_credentials& credentials)
     {
         std::scoped_lock lock(sessions_mutex_);
+        idle_sessions_[type].remove_if([](const auto& s) -> bool { return !s; });
+        busy_sessions_[type].remove_if([](const auto& s) -> bool { return !s; });
         if (idle_sessions_[type].empty()) {
             std::string hostname;
             std::uint16_t port = 0;
@@ -61,21 +80,38 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
             config_.nodes.size();
             std::shared_ptr<http_session> session;
             if (options_.enable_tls) {
-                session = std::make_shared<http_session>(client_id_, ctx_, tls_, credentials, hostname, std::to_string(port));
+                session = std::make_shared<http_session>(type,
+                                                         client_id_,
+                                                         ctx_,
+                                                         tls_,
+                                                         credentials,
+                                                         hostname,
+                                                         std::to_string(port),
+                                                         http_context{ config_, options_, query_cache_ });
             } else {
-                session = std::make_shared<http_session>(client_id_, ctx_, credentials, hostname, std::to_string(port));
+                session = std::make_shared<http_session>(
+                  type, client_id_, ctx_, credentials, hostname, std::to_string(port), http_context{ config_, options_, query_cache_ });
             }
             session->start();
+
             session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {
-                std::scoped_lock inner_lock(self->sessions_mutex_);
-                self->busy_sessions_[type].remove_if([id](const auto& s) -> bool { return s->id() == id; });
-                self->idle_sessions_[type].remove_if([id](const auto& s) -> bool { return s->id() == id; });
+                for (auto& s : self->busy_sessions_[type]) {
+                    if (s && s->id() == id) {
+                        s.reset();
+                    }
+                }
+                for (auto& s : self->idle_sessions_[type]) {
+                    if (s && s->id() == id) {
+                        s.reset();
+                    }
+                }
             });
             busy_sessions_[type].push_back(session);
             return session;
         }
         auto session = idle_sessions_[type].front();
         idle_sessions_[type].pop_front();
+        session->reset_idle();
         busy_sessions_[type].push_back(session);
         return session;
     }
@@ -86,9 +122,32 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
             return session->stop();
         }
         if (!session->is_stopped()) {
+            session->set_idle(options_.idle_http_connection_timeout);
             std::scoped_lock lock(sessions_mutex_);
             spdlog::debug("{} put HTTP session back to idle connections", session->log_prefix());
             idle_sessions_[type].push_back(session);
+            busy_sessions_[type].remove_if([id = session->id()](const auto& s) -> bool { return !s || s->id() == id; });
+        }
+    }
+
+    void close()
+    {
+        {
+            for (auto& sessions : idle_sessions_) {
+                for (auto& s : sessions.second) {
+                    if (s) {
+                        s->reset_idle();
+                        s.reset();
+                    }
+                }
+            }
+        }
+        {
+            for (auto& sessions : busy_sessions_) {
+                for (auto& s : sessions.second) {
+                    s.reset();
+                }
+            }
         }
     }
 
@@ -118,5 +177,6 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
     std::map<service_type, std::list<std::shared_ptr<http_session>>> idle_sessions_{};
     std::size_t next_index_{ 0 };
     std::mutex sessions_mutex_{};
+    query_cache query_cache_{};
 };
 } // namespace couchbase::io

data/ext/couchbase/io/mcbp_command.hxx

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <platform/uuid.h>
+
 #include <io/mcbp_session.hxx>
 #include <io/retry_orchestrator.hxx>
 
@@ -41,12 +43,14 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager,
     std::shared_ptr<io::mcbp_session> session_{};
     mcbp_command_handler handler_{};
     std::shared_ptr<Manager> manager_{};
+    std::string id_;
 
     mcbp_command(asio::io_context& ctx, std::shared_ptr<Manager> manager, Request req)
       : deadline(ctx)
       , retry_backoff(ctx)
       , request(req)
       , manager_(manager)
+      , id_(uuid::to_string(uuid::random()))
     {
     }
 
@@ -85,6 +89,9 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager,
 
     void request_collection_id()
     {
+        if (session_->is_stopped()) {
+            return manager_->map_and_send(this->shared_from_this());
+        }
         protocol::client_request<protocol::get_collection_id_request_body> req;
         req.opaque(session_->next_opaque());
         req.body().collection_path(request.id.collection);
@@ -97,9 +104,9 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager,
               }
               if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
                   if (self->request.id.collection_uid) {
-                      return self->handle_unknown_collection();
+                      return self->invoke_handler(ec);
                   }
-                  return self->invoke_handler(ec);
+                  return self->handle_unknown_collection();
               }
               if (ec) {
                   return self->invoke_handler(ec);
@@ -115,12 +122,13 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager,
     {
         auto backoff = std::chrono::milliseconds(500);
         auto time_left = deadline.expiry() - std::chrono::steady_clock::now();
-        spdlog::debug("{} unknown collection response for \"{}/{}/{}\", time_left={}ms",
+        spdlog::debug(R"({} unknown collection response for "{}/{}/{}", time_left={}ms, id="{}")",
                       session_->log_prefix(),
                       request.id.bucket,
                       request.id.collection,
                       request.id.key,
-                      std::chrono::duration_cast<std::chrono::milliseconds>(time_left).count());
+                      std::chrono::duration_cast<std::chrono::milliseconds>(time_left).count(),
+                      id_);
         if (time_left < backoff) {
             return invoke_handler(std::make_error_code(request.retries.idempotent ? error::common_errc::unambiguous_timeout
                                                                                   : error::common_errc::ambiguous_timeout));
@@ -138,18 +146,19 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager,
     {
         opaque_ = session_->next_opaque();
         request.opaque = *opaque_;
-        if (!request.id.collection_uid) {
+        if (request.id.use_collections && !request.id.collection_uid) {
             if (session_->supports_feature(protocol::hello_feature::collections)) {
                 auto collection_id = session_->get_collection_uid(request.id.collection);
                 if (collection_id) {
                     request.id.collection_uid = *collection_id;
                 } else {
-                    spdlog::debug("{} no cache entry for collection, resolve collection id for \"{}/{}/{}\", timeout={}ms",
+                    spdlog::debug(R"({} no cache entry for collection, resolve collection id for "{}/{}/{}", timeout={}ms, id="{}")",
                                   session_->log_prefix(),
                                   request.id.bucket,
                                   request.id.collection,
                                   request.id.key,
-                                  request.timeout.count());
+                                  request.timeout.count(),
+                                  id_);
                     return request_collection_id();
                 }
             } else {

data/ext/couchbase/io/mcbp_session.hxx

@@ -53,6 +53,7 @@
 #include <origin.hxx>
 #include <errors.hxx>
 #include <version.hxx>
+#include <diagnostics.hxx>
 
 namespace couchbase::io
 {
@@ -202,14 +203,20 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                             return auth_success();
                         }
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={})",
+                                     session_->log_prefix_,
+                                     resp.error_message(),
+                                     resp.opaque());
                         return complete(std::make_error_code(error::network_errc::handshake_failure));
                     }
                 } break;
                 case protocol::client_opcode::sasl_list_mechs: {
                     protocol::client_response<protocol::sasl_list_mechs_response_body> resp(msg);
                     if (resp.status() != protocol::status::success) {
-                        spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={})",
+                                     session_->log_prefix_,
+                                     resp.error_message(),
+                                     resp.opaque());
                         return complete(std::make_error_code(error::common_errc::authentication_failure));
                     }
                 } break;
@@ -232,14 +239,15 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                             req.body().sasl_data(sasl_payload);
                             session_->write_and_flush(req.data());
                         } else {
-                            spdlog::error("{} unable to authenticate: sasl_code={}", session_->log_prefix_, sasl_code);
+                            spdlog::error(
+                              "{} unable to authenticate: (sasl_code={}, opaque={})", session_->log_prefix_, sasl_code, resp.opaque());
                             return complete(std::make_error_code(error::common_errc::authentication_failure));
                         }
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={})",
                                      session_->log_prefix_,
                                      resp.error_message(),
-                                     opcode);
+                                     resp.opaque());
                         return complete(std::make_error_code(error::common_errc::authentication_failure));
                     }
                 } break;
@@ -255,10 +263,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         session_->error_map_.emplace(resp.body().errmap());
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={}, {:n})",
                                      session_->log_prefix_,
                                      resp.error_message(),
-                                     opcode);
+                                     resp.opaque(),
+                                     spdlog::to_hex(msg.header_data()));
                         return complete(std::make_error_code(error::network_errc::protocol_error));
                     }
                 } break;
@@ -267,6 +276,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         spdlog::debug("{} selected bucket: {}", session_->log_prefix_, session_->bucket_name_.value_or(""));
                         session_->bucket_selected_ = true;
+                    } else if (resp.status() == protocol::status::not_found) {
+                        spdlog::debug("{} kv_engine node does not have configuration propagated yet (opcode={}, status={}, opaque={})",
+                                      session_->log_prefix_,
+                                      opcode,
+                                      resp.status(),
+                                      resp.opaque());
+                        return complete(std::make_error_code(error::network_errc::configuration_not_available));
                     } else if (resp.status() == protocol::status::no_access) {
                         spdlog::debug("{} unable to select bucket: {}, probably the bucket does not exist",
                                       session_->log_prefix_,
@@ -274,7 +290,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         session_->bucket_selected_ = false;
                         return complete(std::make_error_code(error::common_errc::bucket_not_found));
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={}, {:n})",
+                                     session_->log_prefix_,
+                                     resp.error_message(),
+                                     resp.opaque(),
+                                     spdlog::to_hex(msg.header_data()));
                         return complete(std::make_error_code(error::common_errc::bucket_not_found));
                     }
                 } break;
@@ -283,6 +303,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         session_->update_configuration(resp.body().config());
                         complete({});
+                    } else if (resp.status() == protocol::status::not_found) {
+                        spdlog::debug("{} kv_engine node does not have configuration propagated yet (opcode={}, status={}, opaque={})",
+                                      session_->log_prefix_,
+                                      opcode,
+                                      resp.status(),
+                                      resp.opaque());
+                        return complete(std::make_error_code(error::network_errc::configuration_not_available));
                     } else if (resp.status() == protocol::status::no_bucket && !session_->bucket_name_) {
                         // bucket-less session, but the server wants bucket
                         session_->supports_gcccp_ = false;
@@ -292,10 +319,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           make_blank_configuration(session_->endpoint_address_, session_->endpoint_.port(), 0));
                         complete({});
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={}, {:n})",
                                      session_->log_prefix_,
                                      resp.error_message(),
-                                     opcode);
+                                     resp.opaque(),
+                                     spdlog::to_hex(msg.header_data()));
                         return complete(std::make_error_code(error::network_errc::protocol_error));
                     }
                 } break;
@@ -353,9 +381,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                                     session_->update_configuration(resp.body().config());
                                 }
                             } else {
-                                spdlog::warn("{} unexpected message status: {}", session_->log_prefix_, resp.error_message());
+                                spdlog::warn("{} unexpected message status: {} (opaque={})",
+                                             session_->log_prefix_,
+                                             resp.error_message(),
+                                             resp.opaque());
                             }
                         } break;
+                        case protocol::client_opcode::get_collections_manifest:
                         case protocol::client_opcode::get_collection_id:
                         case protocol::client_opcode::get:
                         case protocol::client_opcode::get_and_lock:
@@ -376,8 +408,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                             auto handler = session_->command_handlers_.find(opaque);
                             if (handler != session_->command_handlers_.end()) {
                                 auto ec = session_->map_status_code(opcode, status);
-                                spdlog::trace("{} MCBP invoke operation handler: opaque={}, status={}, ec={}",
+                                spdlog::trace("{} MCBP invoke operation handler: opcode={}, opaque={}, status={}, ec={}",
                                               session_->log_prefix_,
+                                              opcode,
                                               opaque,
                                               protocol::status_to_string(status),
                                               ec.message());
@@ -385,14 +418,20 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                                 session_->command_handlers_.erase(handler);
                                 fun(ec, retry_reason::do_not_retry, std::move(msg));
                             } else {
-                                spdlog::debug("{} unexpected orphan response: opcode={}, opaque={}",
+                                spdlog::debug("{} unexpected orphan response: opcode={}, opaque={}, status={}",
                                               session_->log_prefix_,
-                                              msg.header.opcode,
-                                              msg.header.opaque);
+                                              opcode,
+                                              msg.header.opaque,
+                                              protocol::status_to_string(status));
                             }
                         } break;
                         default:
-                            spdlog::warn("{} unexpected client response: {}", session_->log_prefix_, opcode);
+                            spdlog::warn("{} unexpected client response: opcode={}, opaque={}{:a}{:a})",
+                                         session_->log_prefix_,
+                                         opcode,
+                                         msg.header.opaque,
+                                         spdlog::to_hex(msg.header_data()),
+                                         spdlog::to_hex(msg.body));
                     }
                     break;
                 case protocol::magic::server_request:
@@ -400,23 +439,34 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     switch (auto opcode = static_cast<protocol::server_opcode>(msg.header.opcode)) {
                         case protocol::server_opcode::cluster_map_change_notification: {
                             protocol::server_request<protocol::cluster_map_change_notification_request_body> req(msg);
-                            if (session_) {
-                                if ((!req.body().config().bucket.has_value() && req.body().bucket().empty()) ||
+                            std::optional<configuration> config = req.body().config();
+                            if (session_ && config.has_value()) {
+                                if ((!config->bucket.has_value() && req.body().bucket().empty()) ||
                                     (session_->bucket_name_.has_value() && !req.body().bucket().empty() &&
                                      session_->bucket_name_.value() == req.body().bucket())) {
-                                    session_->update_configuration(req.body().config());
+                                    session_->update_configuration(std::move(config.value()));
                                 }
                             }
                         } break;
                         default:
-                            spdlog::warn("{} unexpected server request: {}", session_->log_prefix_, opcode);
+                            spdlog::warn("{} unexpected server request: opcode={:x}, opaque={}{:a}{:a}",
+                                         session_->log_prefix_,
+                                         opcode,
+                                         msg.header.opaque,
+                                         spdlog::to_hex(msg.header_data()),
+                                         spdlog::to_hex(msg.body));
                     }
                     break;
                 case protocol::magic::client_request:
                 case protocol::magic::alt_client_request:
                 case protocol::magic::server_response:
-                    spdlog::warn(
-                      "{} unexpected magic: {}, opcode={}, opaque={}", session_->log_prefix_, magic, msg.header.opcode, msg.header.opaque);
+                    spdlog::warn("{} unexpected magic: {} (opcode={:x}, opaque={}){:a}{:a}",
+                                 session_->log_prefix_,
+                                 magic,
+                                 msg.header.opcode,
+                                 msg.header.opaque,
+                                 spdlog::to_hex(msg.header_data()),
+                                 spdlog::to_hex(msg.body));
                     break;
             }
         }
@@ -453,7 +503,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
       , bucket_name_(std::move(bucket_name))
       , supported_features_(known_features)
     {
-        log_prefix_ = fmt::format("[{}/{}/{}/{}]", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"));
+        log_prefix_ = fmt::format("[{}/{}/{}/{}]", client_id_, id_, stream_->log_prefix(), bucket_name_.value_or("-"));
     }
 
     mcbp_session(const std::string& client_id,
@@ -474,7 +524,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
       , bucket_name_(std::move(bucket_name))
       , supported_features_(known_features)
     {
-        log_prefix_ = fmt::format("[{}/{}/{}/{}]", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"));
+        log_prefix_ = fmt::format("[{}/{}/{}/{}]", client_id_, id_, stream_->log_prefix(), bucket_name_.value_or("-"));
     }
 
     ~mcbp_session()
@@ -487,6 +537,19 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return log_prefix_;
     }
 
+    [[nodiscard]] diag::endpoint_diag_info diag_info() const
+    {
+        return { service_type::kv,
+                 id_,
+                 last_active_.time_since_epoch().count() == 0 ? std::nullopt
+                                                              : std::make_optional(std::chrono::duration_cast<std::chrono::microseconds>(
+                                                                  std::chrono::steady_clock::now() - last_active_)),
+                 fmt::format("{}:{}", endpoint_address_, endpoint_.port()),
+                 fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port()),
+                 state_,
+                 bucket_name_ };
+    }
+
     void bootstrap(std::function<void(std::error_code, configuration)>&& handler, bool retry_on_bucket_not_found = false)
     {
         retry_bootstrap_on_bucket_not_found_ = retry_on_bucket_not_found;
@@ -509,6 +572,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        state_ = diag::endpoint_state::connecting;
+        if (stream_->is_open()) {
+            std::string old_id = stream_->id();
+            stream_->reopen();
+            spdlog::trace(R"({} reopen socket connection "{}" -> "{}", host="{}", port={})",
+                          log_prefix_,
+                          old_id,
+                          stream_->id(),
+                          bootstrap_hostname_,
+                          bootstrap_port_);
+        }
         if (origin_.exhausted()) {
             auto backoff = std::chrono::milliseconds(500);
             spdlog::debug("{} reached the end of list of bootstrap nodes, waiting for {}ms before restart", log_prefix_, backoff.count());
@@ -524,9 +598,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         std::tie(bootstrap_hostname_, bootstrap_port_) = origin_.next_address();
         log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}:{}>",
-                                  stream_->log_prefix(),
                                   client_id_,
                                   id_,
+                                  stream_->log_prefix(),
                                   bucket_name_.value_or("-"),
                                   bootstrap_hostname_,
                                   bootstrap_port_);
@@ -541,6 +615,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return id_;
     }
 
+    [[nodiscard]] bool is_stopped() const
+    {
+        return stopped_;
+    }
+
     void on_stop(std::function<void(io::retry_reason)> handler)
     {
         on_stop_handler_ = std::move(handler);
@@ -551,6 +630,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        state_ = diag::endpoint_state::disconnecting;
         spdlog::debug("{} stop MCBP connection, reason={}", log_prefix_, reason);
         stopped_ = true;
         bootstrap_deadline_.cancel();
@@ -578,6 +658,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             on_stop_handler_(reason);
         }
         on_stop_handler_ = nullptr;
+        state_ = diag::endpoint_state::disconnected;
     }
 
     void write(const std::vector<uint8_t>& buf)
@@ -636,7 +717,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         auto handler = command_handlers_.find(opaque);
         if (handler != command_handlers_.end()) {
-            spdlog::debug("{} MCBP cancel operation, opaque={}, ec={}", log_prefix_, opaque, ec.message());
+            spdlog::debug("{} MCBP cancel operation, opaque={}, ec={} ({})", log_prefix_, opaque, ec.value(), ec.message());
             handler->second(ec, reason, {});
             command_handlers_.erase(handler);
             return true;
@@ -654,7 +735,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return supported_features_;
     }
 
-    [[nodiscard]] bool supports_gcccp()
+    [[nodiscard]] bool supports_gcccp() const
     {
         return supports_gcccp_;
     }
@@ -849,18 +930,46 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
-        if (!config_ || config.rev > config_->rev) {
-            for (auto& node : config.nodes) {
-                if (node.hostname.empty()) {
-                    node.hostname = bootstrap_hostname_;
+        if (config_) {
+            if (config_->vbmap && config.vbmap && config_->vbmap->size() != config.vbmap->size()) {
+                spdlog::debug("{} received a configuration with a different number of vbuckets, ignoring", log_prefix_);
+                return;
+            }
+            if (config_->rev && config.rev) {
+                if (*config_->rev == *config.rev) {
+                    spdlog::debug("{} received a configuration with identical revision (rev={}), ignoring", log_prefix_, *config.rev);
+                    return;
+                }
+                if (*config_->rev > *config.rev) {
+                    spdlog::debug("{} received a configuration with older revision, ignoring", log_prefix_);
+                    return;
                 }
             }
-            config_.emplace(config);
-            spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
-            for (auto& listener : config_listeners_) {
-                listener(*config_);
+        }
+        bool this_node_found = false;
+        for (auto& node : config.nodes) {
+            if (node.hostname.empty()) {
+                node.hostname = bootstrap_hostname_;
+            }
+            if (node.this_node) {
+                this_node_found = true;
+            }
+        }
+        if (!this_node_found) {
+            for (auto& node : config.nodes) {
+                if (node.hostname == bootstrap_hostname_) {
+                    if ((node.services_plain.key_value && std::to_string(node.services_plain.key_value.value()) == bootstrap_port_) ||
+                        (node.services_tls.key_value && std::to_string(node.services_tls.key_value.value()) == bootstrap_port_)) {
+                        node.this_node = true;
+                    }
+                }
             }
         }
+        config_.emplace(config);
+        spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
+        for (auto& listener : config_listeners_) {
+            listener(*config_);
+        }
     }
 
     void handle_not_my_vbucket(io::mcbp_message&& msg)
@@ -887,7 +996,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                               log_prefix_,
                               static_cast<protocol::client_opcode>(msg.header.opcode),
                               msg.header.opaque,
-                              config.rev);
+                              config.rev_str());
                 update_configuration(std::move(config));
             }
         }
@@ -909,8 +1018,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
   private:
     void invoke_bootstrap_handler(std::error_code ec)
     {
+        if (ec == std::make_error_code(error::network_errc::configuration_not_available)) {
+            return initiate_bootstrap();
+        }
         if (retry_bootstrap_on_bucket_not_found_ && ec == std::make_error_code(error::common_errc::bucket_not_found)) {
-            spdlog::debug(R"({} server returned {}, it must be transient condition, retrying)", log_prefix_, ec.message());
+            spdlog::debug(R"({} server returned {} ({}), it must be transient condition, retrying)", log_prefix_, ec.value(), ec.message());
             return initiate_bootstrap();
         }
 
@@ -922,6 +1034,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (ec) {
             return stop(retry_reason::node_not_available);
         }
+        state_ = diag::endpoint_state::connected;
         bootstrapped_ = true;
         handler_ = std::make_unique<normal_handler>(shared_from_this());
         std::scoped_lock lock(pending_buffer_mutex_);
@@ -939,8 +1052,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (ec) {
-            spdlog::error("{} error on resolve: {}", log_prefix_, ec.message());
+            spdlog::error("{} error on resolve: {} ({})", log_prefix_, ec.value(), ec.message());
             return initiate_bootstrap();
         }
         endpoints_ = endpoints;
@@ -954,6 +1068,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (it != endpoints_.end()) {
             spdlog::debug("{} connecting to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
             connection_deadline_.expires_after(timeout_defaults::connect_timeout);
@@ -969,19 +1084,27 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (!stream_->is_open() || ec) {
-            spdlog::warn(
-              "{} unable to connect to {}:{}: {}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port(), ec.message());
+            spdlog::warn("{} unable to connect to {}:{}: {} ({}), is_open={}",
+                         log_prefix_,
+                         it->endpoint().address().to_string(),
+                         it->endpoint().port(),
+                         ec.value(),
+                         ec.message(),
+                         stream_->is_open());
             do_connect(++it);
         } else {
             stream_->set_options();
+            local_endpoint_ = stream_->local_endpoint();
+            local_endpoint_address_ = local_endpoint_.address().to_string();
             endpoint_ = it->endpoint();
             endpoint_address_ = endpoint_.address().to_string();
             spdlog::debug("{} connected to {}:{}", log_prefix_, endpoint_address_, it->endpoint().port());
             log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}/{}:{}>",
-                                      stream_->log_prefix(),
                                       client_id_,
                                       id_,
+                                      stream_->log_prefix(),
                                       bucket_name_.value_or("-"),
                                       bootstrap_hostname_,
                                       endpoint_address_,
@@ -1006,21 +1129,33 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void do_read()
     {
-        if (stopped_) {
-            return;
-        }
-        if (reading_) {
+        if (stopped_ || reading_ || !stream_->is_open()) {
             return;
         }
         reading_ = true;
         stream_->async_read_some(
-          asio::buffer(input_buffer_), [self = shared_from_this()](std::error_code ec, std::size_t bytes_transferred) {
+          asio::buffer(input_buffer_),
+          [self = shared_from_this(), stream_id = stream_->id()](std::error_code ec, std::size_t bytes_transferred) {
               if (ec == asio::error::operation_aborted || self->stopped_) {
                   return;
               }
+              self->last_active_ = std::chrono::steady_clock::now();
               if (ec) {
-                  spdlog::error("{} IO error while reading from the socket: {}, pending_handlers={}",
+                  if (stream_id != self->stream_->id()) {
+                      spdlog::error(
+                        R"({} ignore IO error while reading from the socket: {} ({}), pending_handlers={}, old_id="{}", new_id="{}")",
+                        self->log_prefix_,
+                        ec.value(),
+                        ec.message(),
+                        self->command_handlers_.size(),
+                        stream_id,
+                        self->stream_->id());
+                      return;
+                  }
+                  spdlog::error(R"({} IO error while reading from the socket("{}"): {} ({}), pending_handlers={})",
                                 self->log_prefix_,
+                                self->stream_->id(),
+                                ec.value(),
                                 ec.message(),
                                 self->command_handlers_.size());
                   return self->stop(retry_reason::socket_closed_while_in_flight);
@@ -1045,7 +1180,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           break;
                       case mcbp_parser::need_data:
                           self->reading_ = false;
-                          if (!self->stopped_) {
+                          if (!self->stopped_ && self->stream_->is_open()) {
                               self->do_read();
                           }
                           return;
@@ -1058,7 +1193,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void do_write()
     {
-        if (stopped_) {
+        if (stopped_ || !stream_->is_open()) {
             return;
         }
         std::scoped_lock lock(writing_buffer_mutex_, output_buffer_mutex_);
@@ -1075,9 +1210,12 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             if (ec == asio::error::operation_aborted || self->stopped_) {
                 return;
             }
+            self->last_active_ = std::chrono::steady_clock::now();
             if (ec) {
-                spdlog::error("{} IO error while writing to the socket: {}, pending_handlers={}",
+                spdlog::error(R"({} IO error while writing to the socket("{}"): {} ({}), pending_handlers={})",
                               self->log_prefix_,
+                              self->stream_->id(),
+                              ec.value(),
                               ec.message(),
                               self->command_handlers_.size());
                 return self->stop(retry_reason::socket_closed_while_in_flight);
@@ -1092,7 +1230,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     }
 
     std::string client_id_;
-    std::string id_;
+    const std::string id_;
     asio::io_context& ctx_;
     asio::ip::tcp::resolver resolver_;
     std::unique_ptr<stream_impl> stream_;
@@ -1128,6 +1266,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     std::string bootstrap_port_{};
     asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
     std::string endpoint_address_{};     // cached string with endpoint address
+    asio::ip::tcp::endpoint local_endpoint_{};
+    std::string local_endpoint_address_{};
     asio::ip::tcp::resolver::results_type endpoints_;
     std::vector<protocol::hello_feature> supported_features_;
     std::optional<configuration> config_;
@@ -1137,5 +1277,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     std::atomic_bool reading_{ false };
 
     std::string log_prefix_{};
+    std::chrono::time_point<std::chrono::steady_clock> last_active_{};
+    diag::endpoint_state state_{ diag::endpoint_state::disconnected };
 };
 } // namespace couchbase::io