couchbase 3.0.0.beta.1 → 3.1.0

data/ext/couchbase/io/mcbp_context.hxx

@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2020-2021 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <vector>
+#include <configuration.hxx>
+#include <protocol/hello_feature.hxx>
+
+namespace couchbase
+{
+
+struct mcbp_context {
+    const std::optional<configuration>& config;
+    const std::vector<protocol::hello_feature>& supported_features;
+
+    [[nodiscard]] bool supports_feature(protocol::hello_feature feature)
+    {
+        return std::find(supported_features.begin(), supported_features.end(), feature) != supported_features.end();
+    }
+};
+
+} // namespace couchbase

data/ext/couchbase/io/mcbp_message.hxx

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
 /*
- *     Copyright 2020 Couchbase, Inc.
+ *   Copyright 2020-2021 Couchbase, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.

data/ext/couchbase/io/mcbp_parser.hxx

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
 /*
- *     Copyright 2020 Couchbase, Inc.
+ *   Copyright 2020-2021 Couchbase, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.

data/ext/couchbase/io/mcbp_session.hxx

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
 /*
- *     Copyright 2020 Couchbase, Inc.
+ *   Copyright 2020-2021 Couchbase, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -28,6 +28,8 @@
 #include <io/mcbp_message.hxx>
 #include <io/mcbp_parser.hxx>
 #include <io/streams.hxx>
+#include <io/retry_orchestrator.hxx>
+#include <io/mcbp_context.hxx>
 
 #include <timeout_defaults.hxx>
 
@@ -35,6 +37,7 @@
 #include <protocol/client_request.hxx>
 #include <protocol/client_response.hxx>
 #include <protocol/server_request.hxx>
+#include <protocol/cmd_noop.hxx>
 #include <protocol/cmd_hello.hxx>
 #include <protocol/cmd_sasl_list_mechs.hxx>
 #include <protocol/cmd_sasl_auth.hxx>
@@ -52,6 +55,7 @@
 #include <origin.hxx>
 #include <errors.hxx>
 #include <version.hxx>
+#include <diagnostics.hxx>
 
 namespace couchbase::io
 {
@@ -120,17 +124,21 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
         explicit bootstrap_handler(std::shared_ptr<mcbp_session> session)
           : session_(session)
-          , sasl_([origin = session_->origin_]() -> std::string { return origin.get_username(); },
-                  [origin = session_->origin_]() -> std::string { return origin.get_password(); },
-                  { "SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1", "PLAIN" })
+          , sasl_([origin = session_->origin_]() -> std::string { return origin.username(); },
+                  [origin = session_->origin_]() -> std::string { return origin.password(); },
+                  session_->origin_.credentials().allowed_sasl_mechanisms)
         {
             tao::json::value user_agent{
-                { "a",
-                  fmt::format(
-                    "ruby/{}.{}.{}/{}", BACKEND_VERSION_MAJOR, BACKEND_VERSION_MINOR, BACKEND_VERSION_PATCH, BACKEND_GIT_REVISION) },
-                { "i", fmt::format("{}/{}", session_->client_id_, session_->id_) }
+                { "a", couchbase::sdk_id() },
+                { "i", fmt::format("{}/{}", session_->client_id_, session_->id_) },
             };
             protocol::client_request<protocol::hello_request_body> hello_req;
+            if (session_->origin_.options().enable_unordered_execution) {
+                hello_req.body().enable_unordered_execution();
+            }
+            if (session_->origin_.options().enable_clustermap_notification) {
+                hello_req.body().enable_clustermap_change_notification();
+            }
             hello_req.opaque(session_->next_opaque());
             hello_req.body().user_agent(tao::json::to_string(user_agent));
             spdlog::debug("{} user_agent={}, requested_features=[{}]",
@@ -139,18 +147,20 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           fmt::join(hello_req.body().features(), ", "));
             session_->write(hello_req.data());
 
-            protocol::client_request<protocol::sasl_list_mechs_request_body> list_req;
-            list_req.opaque(session_->next_opaque());
-            session_->write(list_req.data());
-
-            protocol::client_request<protocol::sasl_auth_request_body> auth_req;
-            sasl::error sasl_code;
-            std::string_view sasl_payload;
-            std::tie(sasl_code, sasl_payload) = sasl_.start();
-            auth_req.opaque(session_->next_opaque());
-            auth_req.body().mechanism(sasl_.get_name());
-            auth_req.body().sasl_data(sasl_payload);
-            session_->write(auth_req.data());
+            if (!session->origin_.credentials().uses_certificate()) {
+                protocol::client_request<protocol::sasl_list_mechs_request_body> list_req;
+                list_req.opaque(session_->next_opaque());
+                session_->write(list_req.data());
+
+                protocol::client_request<protocol::sasl_auth_request_body> auth_req;
+                sasl::error sasl_code;
+                std::string_view sasl_payload;
+                std::tie(sasl_code, sasl_payload) = sasl_.start();
+                auth_req.opaque(session_->next_opaque());
+                auth_req.body().mechanism(sasl_.get_name());
+                auth_req.body().sasl_data(sasl_payload);
+                session_->write(auth_req.data());
+            }
 
             session_->flush();
         }
@@ -194,15 +204,25 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         session_->supported_features_ = resp.body().supported_features();
                         spdlog::debug("{} supported_features=[{}]", session_->log_prefix_, fmt::join(session_->supported_features_, ", "));
+                        if (session_->origin_.credentials().uses_certificate()) {
+                            spdlog::debug("{} skip SASL authentication, because TLS certificate was specified", session_->log_prefix_);
+                            return auth_success();
+                        }
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={})",
+                                     session_->log_prefix_,
+                                     resp.error_message(),
+                                     resp.opaque());
                         return complete(std::make_error_code(error::network_errc::handshake_failure));
                     }
                 } break;
                 case protocol::client_opcode::sasl_list_mechs: {
                     protocol::client_response<protocol::sasl_list_mechs_response_body> resp(msg);
                     if (resp.status() != protocol::status::success) {
-                        spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={})",
+                                     session_->log_prefix_,
+                                     resp.error_message(),
+                                     resp.opaque());
                         return complete(std::make_error_code(error::common_errc::authentication_failure));
                     }
                 } break;
@@ -225,14 +245,15 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                             req.body().sasl_data(sasl_payload);
                             session_->write_and_flush(req.data());
                         } else {
-                            spdlog::error("{} unable to authenticate: sasl_code={}", session_->log_prefix_, sasl_code);
+                            spdlog::error(
+                              "{} unable to authenticate: (sasl_code={}, opaque={})", session_->log_prefix_, sasl_code, resp.opaque());
                             return complete(std::make_error_code(error::common_errc::authentication_failure));
                         }
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={})",
                                      session_->log_prefix_,
                                      resp.error_message(),
-                                     opcode);
+                                     resp.opaque());
                         return complete(std::make_error_code(error::common_errc::authentication_failure));
                     }
                 } break;
@@ -246,12 +267,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 case protocol::client_opcode::get_error_map: {
                     protocol::client_response<protocol::get_error_map_response_body> resp(msg);
                     if (resp.status() == protocol::status::success) {
-                        session_->errmap_.emplace(resp.body().errmap());
+                        session_->error_map_.emplace(resp.body().errmap());
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={}, {:n})",
                                      session_->log_prefix_,
                                      resp.error_message(),
-                                     opcode);
+                                     resp.opaque(),
+                                     spdlog::to_hex(msg.header_data()));
                         return complete(std::make_error_code(error::network_errc::protocol_error));
                     }
                 } break;
@@ -260,6 +282,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         spdlog::debug("{} selected bucket: {}", session_->log_prefix_, session_->bucket_name_.value_or(""));
                         session_->bucket_selected_ = true;
+                    } else if (resp.status() == protocol::status::not_found) {
+                        spdlog::debug("{} kv_engine node does not have configuration propagated yet (opcode={}, status={}, opaque={})",
+                                      session_->log_prefix_,
+                                      opcode,
+                                      resp.status(),
+                                      resp.opaque());
+                        return complete(std::make_error_code(error::network_errc::configuration_not_available));
                     } else if (resp.status() == protocol::status::no_access) {
                         spdlog::debug("{} unable to select bucket: {}, probably the bucket does not exist",
                                       session_->log_prefix_,
@@ -267,7 +296,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         session_->bucket_selected_ = false;
                         return complete(std::make_error_code(error::common_errc::bucket_not_found));
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={}, {:n})",
+                                     session_->log_prefix_,
+                                     resp.error_message(),
+                                     resp.opaque(),
+                                     spdlog::to_hex(msg.header_data()));
                         return complete(std::make_error_code(error::common_errc::bucket_not_found));
                     }
                 } break;
@@ -276,6 +309,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         session_->update_configuration(resp.body().config());
                         complete({});
+                    } else if (resp.status() == protocol::status::not_found) {
+                        spdlog::debug("{} kv_engine node does not have configuration propagated yet (opcode={}, status={}, opaque={})",
+                                      session_->log_prefix_,
+                                      opcode,
+                                      resp.status(),
+                                      resp.opaque());
+                        return complete(std::make_error_code(error::network_errc::configuration_not_available));
                     } else if (resp.status() == protocol::status::no_bucket && !session_->bucket_name_) {
                         // bucket-less session, but the server wants bucket
                         session_->supports_gcccp_ = false;
@@ -285,10 +325,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           make_blank_configuration(session_->endpoint_address_, session_->endpoint_.port(), 0));
                         complete({});
                     } else {
-                        spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
+                        spdlog::warn("{} unexpected message status during bootstrap: {} (opaque={}, {:n})",
                                      session_->log_prefix_,
                                      resp.error_message(),
-                                     opcode);
+                                     resp.opaque(),
+                                     spdlog::to_hex(msg.header_data()));
                         return complete(std::make_error_code(error::network_errc::protocol_error));
                     }
                 } break;
@@ -346,9 +387,14 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                                     session_->update_configuration(resp.body().config());
                                 }
                             } else {
-                                spdlog::warn("{} unexpected message status: {}", session_->log_prefix_, resp.error_message());
+                                spdlog::warn("{} unexpected message status: {} (opaque={})",
+                                             session_->log_prefix_,
+                                             resp.error_message(),
+                                             resp.opaque());
                             }
                         } break;
+                        case protocol::client_opcode::noop:
+                        case protocol::client_opcode::get_collections_manifest:
                         case protocol::client_opcode::get_collection_id:
                         case protocol::client_opcode::get:
                         case protocol::client_opcode::get_and_lock:
@@ -357,6 +403,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         case protocol::client_opcode::insert:
                         case protocol::client_opcode::replace:
                         case protocol::client_opcode::upsert:
+                        case protocol::client_opcode::append:
+                        case protocol::client_opcode::prepend:
                         case protocol::client_opcode::remove:
                         case protocol::client_opcode::observe:
                         case protocol::client_opcode::unlock:
@@ -366,26 +414,36 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         case protocol::client_opcode::subdoc_multi_mutation: {
                             std::uint32_t opaque = msg.header.opaque;
                             std::uint16_t status = ntohs(msg.header.specific);
+                            session_->command_handlers_mutex_.lock();
                             auto handler = session_->command_handlers_.find(opaque);
-                            if (handler != session_->command_handlers_.end()) {
+                            if (handler != session_->command_handlers_.end() && handler->second) {
                                 auto ec = session_->map_status_code(opcode, status);
-                                spdlog::debug("{} MCBP invoke operation handler, opaque={}, status={}, ec={}",
+                                spdlog::trace("{} MCBP invoke operation handler: opcode={}, opaque={}, status={}, ec={}",
                                               session_->log_prefix_,
+                                              opcode,
                                               opaque,
-                                              status,
+                                              protocol::status_to_string(status),
                                               ec.message());
-                                auto fun = handler->second;
+                                auto fun = std::move(handler->second);
                                 session_->command_handlers_.erase(handler);
-                                fun(ec, std::move(msg));
+                                session_->command_handlers_mutex_.unlock();
+                                fun(ec, retry_reason::do_not_retry, std::move(msg));
                             } else {
-                                spdlog::debug("{} unexpected orphan response opcode={}, opaque={}",
+                                session_->command_handlers_mutex_.unlock();
+                                spdlog::debug("{} unexpected orphan response: opcode={}, opaque={}, status={}",
                                               session_->log_prefix_,
-                                              msg.header.opcode,
-                                              msg.header.opaque);
+                                              opcode,
+                                              msg.header.opaque,
+                                              protocol::status_to_string(status));
                             }
                         } break;
                         default:
-                            spdlog::warn("{} unexpected client response: {}", session_->log_prefix_, opcode);
+                            spdlog::warn("{} unexpected client response: opcode={}, opaque={}{:a}{:a})",
+                                         session_->log_prefix_,
+                                         opcode,
+                                         msg.header.opaque,
+                                         spdlog::to_hex(msg.header_data()),
+                                         spdlog::to_hex(msg.body));
                     }
                     break;
                 case protocol::magic::server_request:
@@ -393,23 +451,34 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     switch (auto opcode = static_cast<protocol::server_opcode>(msg.header.opcode)) {
                         case protocol::server_opcode::cluster_map_change_notification: {
                             protocol::server_request<protocol::cluster_map_change_notification_request_body> req(msg);
-                            if (session_) {
-                                if ((!req.body().config().bucket.has_value() && req.body().bucket().empty()) ||
+                            std::optional<configuration> config = req.body().config();
+                            if (session_ && config.has_value()) {
+                                if ((!config->bucket.has_value() && req.body().bucket().empty()) ||
                                     (session_->bucket_name_.has_value() && !req.body().bucket().empty() &&
                                      session_->bucket_name_.value() == req.body().bucket())) {
-                                    session_->update_configuration(req.body().config());
+                                    session_->update_configuration(std::move(config.value()));
                                 }
                             }
                         } break;
                         default:
-                            spdlog::warn("{} unexpected server request: {}", session_->log_prefix_, opcode);
+                            spdlog::warn("{} unexpected server request: opcode={:x}, opaque={}{:a}{:a}",
+                                         session_->log_prefix_,
+                                         opcode,
+                                         msg.header.opaque,
+                                         spdlog::to_hex(msg.header_data()),
+                                         spdlog::to_hex(msg.body));
                     }
                     break;
                 case protocol::magic::client_request:
                 case protocol::magic::alt_client_request:
                 case protocol::magic::server_response:
-                    spdlog::warn(
-                      "{} unexpected magic: {}, opcode={}, opaque={}", session_->log_prefix_, magic, msg.header.opcode, msg.header.opaque);
+                    spdlog::warn("{} unexpected magic: {} (opcode={:x}, opaque={}){:a}{:a}",
+                                 session_->log_prefix_,
+                                 magic,
+                                 msg.header.opcode,
+                                 msg.header.opaque,
+                                 spdlog::to_hex(msg.header_data()),
+                                 spdlog::to_hex(msg.body));
                     break;
             }
         }
@@ -446,7 +515,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
       , bucket_name_(std::move(bucket_name))
       , supported_features_(known_features)
     {
-        log_prefix_ = fmt::format("[{}/{}/{}/{}]", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"));
+        log_prefix_ = fmt::format("[{}/{}/{}/{}]", client_id_, id_, stream_->log_prefix(), bucket_name_.value_or("-"));
     }
 
     mcbp_session(const std::string& client_id,
@@ -467,12 +536,12 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
       , bucket_name_(std::move(bucket_name))
       , supported_features_(known_features)
     {
-        log_prefix_ = fmt::format("[{}/{}/{}/{}]", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"));
+        log_prefix_ = fmt::format("[{}/{}/{}/{}]", client_id_, id_, stream_->log_prefix(), bucket_name_.value_or("-"));
     }
 
     ~mcbp_session()
     {
-        stop();
+        stop(retry_reason::do_not_retry);
     }
 
     [[nodiscard]] const std::string& log_prefix() const
@@ -480,8 +549,70 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return log_prefix_;
     }
 
-    void bootstrap(std::function<void(std::error_code, configuration)>&& handler)
+    std::string remote_address() const
+    {
+        if (endpoint_.protocol() == asio::ip::tcp::v6()) {
+            return fmt::format("[{}]:{}", endpoint_address_, endpoint_.port());
+        }
+        return fmt::format("{}:{}", endpoint_address_, endpoint_.port());
+    }
+
+    std::string local_address() const
     {
+        if (endpoint_.protocol() == asio::ip::tcp::v6()) {
+            return fmt::format("[{}]:{}", local_endpoint_address_, local_endpoint_.port());
+        }
+        return fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port());
+    }
+
+    [[nodiscard]] diag::endpoint_diag_info diag_info() const
+    {
+        return { service_type::kv,
+                 id_,
+                 last_active_.time_since_epoch().count() == 0 ? std::nullopt
+                                                              : std::make_optional(std::chrono::duration_cast<std::chrono::microseconds>(
+                                                                  std::chrono::steady_clock::now() - last_active_)),
+                 remote_address(),
+                 local_address(),
+                 state_,
+                 bucket_name_ };
+    }
+
+    template<typename Handler>
+    void ping(Handler&& handler)
+    {
+        protocol::client_request<protocol::mcbp_noop_request_body> req;
+        req.opaque(next_opaque());
+        write_and_subscribe(req.opaque(),
+                            req.data(false),
+                            [start = std::chrono::steady_clock::now(), self = shared_from_this(), handler](
+                              std::error_code ec, retry_reason reason, io::mcbp_message&& /* msg */) {
+                                diag::ping_state state = diag::ping_state::ok;
+                                std::optional<std::string> error{};
+                                if (ec) {
+                                    state = diag::ping_state::error;
+                                    error.emplace(fmt::format("code={}, message={}, reason={}", ec.value(), ec.message(), reason));
+                                }
+                                handler(diag::endpoint_ping_info{
+                                  service_type::kv,
+                                  self->id_,
+                                  std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start),
+                                  self->remote_address(),
+                                  self->local_address(),
+                                  state,
+                                  self->bucket_name_,
+                                  error });
+                            });
+    }
+
+    [[nodiscard]] mcbp_context context() const
+    {
+        return { config_, supported_features_ };
+    }
+
+    void bootstrap(std::function<void(std::error_code, configuration)>&& handler, bool retry_on_bucket_not_found = false)
+    {
+        retry_bootstrap_on_bucket_not_found_ = retry_on_bucket_not_found;
         bootstrap_handler_ = std::move(handler);
         bootstrap_deadline_.expires_after(timeout_defaults::bootstrap_timeout);
         bootstrap_deadline_.async_wait([self = shared_from_this()](std::error_code ec) {
@@ -491,7 +622,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             spdlog::warn("{} unable to bootstrap in time", self->log_prefix_);
             self->bootstrap_handler_(std::make_error_code(error::common_errc::unambiguous_timeout), {});
             self->bootstrap_handler_ = nullptr;
-            self->stop();
+            self->stop(retry_reason::socket_closed_while_in_flight);
         });
         initiate_bootstrap();
     }
@@ -501,6 +632,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        state_ = diag::endpoint_state::connecting;
+        if (stream_->is_open()) {
+            std::string old_id = stream_->id();
+            stream_->reopen();
+            spdlog::trace(R"({} reopen socket connection "{}" -> "{}", host="{}", port={})",
+                          log_prefix_,
+                          old_id,
+                          stream_->id(),
+                          bootstrap_hostname_,
+                          bootstrap_port_);
+        }
         if (origin_.exhausted()) {
             auto backoff = std::chrono::milliseconds(500);
             spdlog::debug("{} reached the end of list of bootstrap nodes, waiting for {}ms before restart", log_prefix_, backoff.count());
@@ -514,13 +656,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             });
             return;
         }
-        std::string service;
-        std::tie(bootstrap_hostname_, service) = origin_.next_address();
-        log_prefix_ = fmt::format(
-          "[{}/{}/{}/{}] <{}:{}>", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"), bootstrap_hostname_, service);
+        std::tie(bootstrap_hostname_, bootstrap_port_) = origin_.next_address();
+        log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}:{}>",
+                                  client_id_,
+                                  id_,
+                                  stream_->log_prefix(),
+                                  bucket_name_.value_or("-"),
+                                  bootstrap_hostname_,
+                                  bootstrap_port_);
         spdlog::debug("{} attempt to establish MCBP connection", log_prefix_);
         resolver_.async_resolve(bootstrap_hostname_,
-                                service,
+                                bootstrap_port_,
                                 std::bind(&mcbp_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
     }
 
@@ -529,11 +675,23 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return id_;
     }
 
-    void stop()
+    [[nodiscard]] bool is_stopped() const
+    {
+        return stopped_;
+    }
+
+    void on_stop(std::function<void(io::retry_reason)> handler)
+    {
+        on_stop_handler_ = std::move(handler);
+    }
+
+    void stop(retry_reason reason)
     {
         if (stopped_) {
             return;
         }
+        state_ = diag::endpoint_state::disconnecting;
+        spdlog::debug("{} stop MCBP connection, reason={}", log_prefix_, reason);
         stopped_ = true;
         bootstrap_deadline_.cancel();
         connection_deadline_.cancel();
@@ -550,11 +708,24 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (handler_) {
             handler_->stop();
         }
-        for (auto& handler : command_handlers_) {
-            spdlog::debug("{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
-            handler.second(ec, {});
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            for (auto& handler : command_handlers_) {
+                if (handler.second) {
+                    spdlog::debug(
+                      "{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
+                    auto fun = std::move(handler.second);
+                    fun(ec, reason, {});
+                }
+            }
+            command_handlers_.clear();
+        }
+        config_listeners_.clear();
+        if (on_stop_handler_) {
+            on_stop_handler_(reason);
         }
-        command_handlers_.clear();
+        on_stop_handler_ = nullptr;
+        state_ = diag::endpoint_state::disconnected;
     }
 
     void write(const std::vector<uint8_t>& buf)
@@ -564,7 +735,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         std::uint32_t opaque{ 0 };
         std::memcpy(&opaque, buf.data() + 12, sizeof(opaque));
-        spdlog::debug("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
+        spdlog::trace("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
         SPDLOG_TRACE("{} MCBP send, opaque={}{:a}", log_prefix_, opaque, spdlog::to_hex(data));
         std::scoped_lock lock(output_buffer_mutex_);
         output_buffer_.push_back(buf);
@@ -589,33 +760,45 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void write_and_subscribe(uint32_t opaque,
                              std::vector<std::uint8_t>& data,
-                             std::function<void(std::error_code, io::mcbp_message&&)> handler)
+                             std::function<void(std::error_code, retry_reason, io::mcbp_message&&)> handler)
     {
         if (stopped_) {
-            spdlog::warn("{} MCBP cancel operation, while trying to write to closed session opaque={}", log_prefix_, opaque);
-            handler(std::make_error_code(error::common_errc::request_canceled), {});
+            spdlog::warn("{} MCBP cancel operation, while trying to write to closed session, opaque={}", log_prefix_, opaque);
+            handler(std::make_error_code(error::common_errc::request_canceled), retry_reason::socket_closed_while_in_flight, {});
             return;
         }
-        command_handlers_.emplace(opaque, std::move(handler));
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            command_handlers_.emplace(opaque, std::move(handler));
+        }
         if (bootstrapped_ && stream_->is_open()) {
             write_and_flush(data);
         } else {
+            spdlog::debug("{} the stream is not ready yet, put the message into pending buffer, opaque={}", log_prefix_, opaque);
             std::scoped_lock lock(pending_buffer_mutex_);
             pending_buffer_.push_back(data);
         }
     }
 
-    void cancel(uint32_t opaque, std::error_code ec)
+    [[nodiscard]] bool cancel(uint32_t opaque, std::error_code ec, retry_reason reason)
     {
         if (stopped_) {
-            return;
+            return false;
         }
+        command_handlers_mutex_.lock();
         auto handler = command_handlers_.find(opaque);
         if (handler != command_handlers_.end()) {
-            spdlog::debug("{} MCBP cancel operation, opaque={}, ec={}", log_prefix_, opaque, ec.message());
-            handler->second(ec, {});
-            command_handlers_.erase(handler);
+            spdlog::debug("{} MCBP cancel operation, opaque={}, ec={} ({})", log_prefix_, opaque, ec.value(), ec.message());
+            if (handler->second) {
+                auto fun = std::move(handler->second);
+                command_handlers_.erase(handler);
+                command_handlers_mutex_.unlock();
+                fun(ec, reason, {});
+                return true;
+            }
         }
+        command_handlers_mutex_.unlock();
+        return false;
     }
 
     [[nodiscard]] bool supports_feature(protocol::hello_feature feature)
@@ -628,7 +811,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return supported_features_;
     }
 
-    [[nodiscard]] bool supports_gcccp()
+    [[nodiscard]] bool supports_gcccp() const
     {
         return supports_gcccp_;
     }
@@ -654,6 +837,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return bootstrap_hostname_;
     }
 
+    [[nodiscard]] const std::string& bootstrap_port() const
+    {
+        return bootstrap_port_;
+    }
+
     [[nodiscard]] uint32_t next_opaque()
     {
         return ++opaque_;
@@ -708,7 +896,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 return std::make_error_code(error::common_errc::internal_server_failure);
 
             case protocol::status::busy:
-            case protocol::status::temp_failure:
+            case protocol::status::temporary_failure:
             case protocol::status::no_memory:
             case protocol::status::not_initialized:
                 return std::make_error_code(error::common_errc::temporary_failure);
@@ -793,23 +981,100 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 break;
         }
         // FIXME: use error map here
-        spdlog::warn("{} unknown status code: {} (opcode={})", log_prefix_, status, opcode);
+        spdlog::warn("{} unknown status code: {} (opcode={})", log_prefix_, protocol::status_to_string(status), opcode);
         return std::make_error_code(error::network_errc::protocol_error);
     }
 
+    std::optional<error_map::error_info> decode_error_code(std::uint16_t code)
+    {
+        if (error_map_) {
+            auto info = error_map_->errors.find(code);
+            if (info != error_map_->errors.end()) {
+                return info->second;
+            }
+        }
+        return {};
+    }
+
+    void on_configuration_update(std::function<void(const configuration&)> handler)
+    {
+        config_listeners_.emplace_back(std::move(handler));
+    }
+
     void update_configuration(configuration&& config)
     {
         if (stopped_) {
             return;
         }
-        if (!config_ || config.rev > config_->rev) {
+        if (config_) {
+            if (config_->vbmap && config.vbmap && config_->vbmap->size() != config.vbmap->size()) {
+                spdlog::debug("{} received a configuration with a different number of vbuckets, ignoring", log_prefix_);
+                return;
+            }
+            if (config_->rev && config.rev) {
+                if (*config_->rev == *config.rev) {
+                    spdlog::trace("{} received a configuration with identical revision (rev={}), ignoring", log_prefix_, *config.rev);
+                    return;
+                }
+                if (*config_->rev > *config.rev) {
+                    spdlog::debug("{} received a configuration with older revision, ignoring", log_prefix_);
+                    return;
+                }
+            }
+        }
+        bool this_node_found = false;
+        for (auto& node : config.nodes) {
+            if (node.hostname.empty()) {
+                node.hostname = bootstrap_hostname_;
+            }
+            if (node.this_node) {
+                this_node_found = true;
+            }
+        }
+        if (!this_node_found) {
             for (auto& node : config.nodes) {
-                if (node.this_node && node.hostname.empty()) {
-                    node.hostname = endpoint_address_;
+                if (node.hostname == bootstrap_hostname_) {
+                    if ((node.services_plain.key_value && std::to_string(node.services_plain.key_value.value()) == bootstrap_port_) ||
+                        (node.services_tls.key_value && std::to_string(node.services_tls.key_value.value()) == bootstrap_port_)) {
+                        node.this_node = true;
+                    }
                 }
             }
-            config_.emplace(config);
-            spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
+        }
+        config_.emplace(config);
+        spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
+        for (auto& listener : config_listeners_) {
+            listener(*config_);
+        }
+    }
+
+    void handle_not_my_vbucket(io::mcbp_message&& msg)
+    {
+        if (stopped_) {
+            return;
+        }
+        Expects(msg.header.magic == static_cast<std::uint8_t>(protocol::magic::alt_client_response) ||
+                msg.header.magic == static_cast<std::uint8_t>(protocol::magic::client_response));
+        if (protocol::has_json_datatype(msg.header.datatype)) {
+            auto magic = static_cast<protocol::magic>(msg.header.magic);
+            uint8_t extras_size = msg.header.extlen;
+            uint8_t framing_extras_size = 0;
+            uint16_t key_size = htons(msg.header.keylen);
+            if (magic == protocol::magic::alt_client_response) {
+                framing_extras_size = static_cast<std::uint8_t>(msg.header.keylen >> 8U);
+                key_size = msg.header.keylen & 0xffU;
+            }
+
+            std::vector<uint8_t>::difference_type offset = framing_extras_size + key_size + extras_size;
+            if (ntohl(msg.header.bodylen) - offset > 0) {
+                auto config = protocol::parse_config(msg.body.begin() + offset, msg.body.end());
+                spdlog::debug("{} received not_my_vbucket status for {}, opaque={} with config rev={} in the payload",
+                              log_prefix_,
+                              static_cast<protocol::client_opcode>(msg.header.opcode),
+                              msg.header.opaque,
+                              config.rev_str());
+                update_configuration(std::move(config));
+            }
         }
     }
 
@@ -829,14 +1094,23 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
   private:
     void invoke_bootstrap_handler(std::error_code ec)
     {
+        if (ec == std::make_error_code(error::network_errc::configuration_not_available)) {
+            return initiate_bootstrap();
+        }
+        if (retry_bootstrap_on_bucket_not_found_ && ec == std::make_error_code(error::common_errc::bucket_not_found)) {
+            spdlog::debug(R"({} server returned {} ({}), it must be transient condition, retrying)", log_prefix_, ec.value(), ec.message());
+            return initiate_bootstrap();
+        }
+
         if (!bootstrapped_ && bootstrap_handler_) {
             bootstrap_deadline_.cancel();
             bootstrap_handler_(ec, config_.value_or(configuration{}));
             bootstrap_handler_ = nullptr;
         }
         if (ec) {
-            return stop();
+            return stop(retry_reason::node_not_available);
         }
+        state_ = diag::endpoint_state::connected;
         bootstrapped_ = true;
         handler_ = std::make_unique<normal_handler>(shared_from_this());
         std::scoped_lock lock(pending_buffer_mutex_);
@@ -854,8 +1128,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (ec) {
-            spdlog::error("{} error on resolve: {}", log_prefix_, ec.message());
+            spdlog::error("{} error on resolve: {} ({})", log_prefix_, ec.value(), ec.message());
             return initiate_bootstrap();
         }
         endpoints_ = endpoints;
@@ -869,6 +1144,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (it != endpoints_.end()) {
             spdlog::debug("{} connecting to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
             connection_deadline_.expires_after(timeout_defaults::connect_timeout);
@@ -884,19 +1160,27 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (!stream_->is_open() || ec) {
-            spdlog::warn(
-              "{} unable to connect to {}:{}: {}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port(), ec.message());
+            spdlog::warn("{} unable to connect to {}:{}: {} ({}), is_open={}",
+                         log_prefix_,
+                         it->endpoint().address().to_string(),
+                         it->endpoint().port(),
+                         ec.value(),
+                         ec.message(),
+                         stream_->is_open());
             do_connect(++it);
         } else {
             stream_->set_options();
+            local_endpoint_ = stream_->local_endpoint();
+            local_endpoint_address_ = local_endpoint_.address().to_string();
             endpoint_ = it->endpoint();
             endpoint_address_ = endpoint_.address().to_string();
             spdlog::debug("{} connected to {}:{}", log_prefix_, endpoint_address_, it->endpoint().port());
             log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}/{}:{}>",
-                                      stream_->log_prefix(),
                                       client_id_,
                                       id_,
+                                      stream_->log_prefix(),
                                       bucket_name_.value_or("-"),
                                       bootstrap_hostname_,
                                       endpoint_address_,
@@ -921,21 +1205,33 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void do_read()
     {
-        if (stopped_) {
-            return;
-        }
-        if (reading_) {
+        if (stopped_ || reading_ || !stream_->is_open()) {
             return;
         }
         reading_ = true;
         stream_->async_read_some(
-          asio::buffer(input_buffer_), [self = shared_from_this()](std::error_code ec, std::size_t bytes_transferred) {
+          asio::buffer(input_buffer_),
+          [self = shared_from_this(), stream_id = stream_->id()](std::error_code ec, std::size_t bytes_transferred) {
               if (ec == asio::error::operation_aborted || self->stopped_) {
                   return;
               }
+              self->last_active_ = std::chrono::steady_clock::now();
               if (ec) {
-                  spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
-                  return self->stop();
+                  if (stream_id != self->stream_->id()) {
+                      spdlog::error(R"({} ignore IO error while reading from the socket: {} ({}), old_id="{}", new_id="{}")",
+                                    self->log_prefix_,
+                                    ec.value(),
+                                    ec.message(),
+                                    stream_id,
+                                    self->stream_->id());
+                      return;
+                  }
+                  spdlog::error(R"({} IO error while reading from the socket("{}"): {} ({}))",
+                                self->log_prefix_,
+                                self->stream_->id(),
+                                ec.value(),
+                                ec.message());
+                  return self->stop(retry_reason::socket_closed_while_in_flight);
               }
               self->parser_.feed(self->input_buffer_.data(), self->input_buffer_.data() + ssize_t(bytes_transferred));
 
@@ -943,7 +1239,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                   mcbp_message msg{};
                   switch (self->parser_.next(msg)) {
                       case mcbp_parser::ok:
-                          spdlog::debug(
+                          spdlog::trace(
                             "{} MCBP recv, opaque={}, {:n}", self->log_prefix_, msg.header.opaque, spdlog::to_hex(msg.header_data()));
                           SPDLOG_TRACE("{} MCBP recv, opaque={}{:a}{:a}",
                                        self->log_prefix_,
@@ -957,12 +1253,12 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           break;
                       case mcbp_parser::need_data:
                           self->reading_ = false;
-                          if (!self->stopped_) {
+                          if (!self->stopped_ && self->stream_->is_open()) {
                               self->do_read();
                           }
                           return;
                       case mcbp_parser::failure:
-                          return self->stop();
+                          return self->stop(retry_reason::kv_temporary_failure);
                   }
               }
           });
@@ -970,7 +1266,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void do_write()
     {
-        if (stopped_) {
+        if (stopped_ || !stream_->is_open()) {
             return;
         }
         std::scoped_lock lock(writing_buffer_mutex_, output_buffer_mutex_);
@@ -987,9 +1283,14 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             if (ec == asio::error::operation_aborted || self->stopped_) {
                 return;
             }
+            self->last_active_ = std::chrono::steady_clock::now();
             if (ec) {
-                spdlog::error("{} IO error while writing to the socket: {}", self->log_prefix_, ec.message());
-                return self->stop();
+                spdlog::error(R"({} IO error while writing to the socket("{}"): {} ({}))",
+                              self->log_prefix_,
+                              self->stream_->id(),
+                              ec.value(),
+                              ec.message());
+                return self->stop(retry_reason::socket_closed_while_in_flight);
             }
             {
                 std::scoped_lock inner_lock(self->writing_buffer_mutex_);
@@ -1001,7 +1302,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     }
 
     std::string client_id_;
-    std::string id_;
+    const std::string id_;
     asio::io_context& ctx_;
     asio::ip::tcp::resolver resolver_;
     std::unique_ptr<stream_impl> stream_;
@@ -1013,13 +1314,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     mcbp_parser parser_;
     std::unique_ptr<message_handler> handler_;
     std::function<void(std::error_code, const configuration&)> bootstrap_handler_{};
-    std::map<uint32_t, std::function<void(std::error_code, io::mcbp_message&&)>> command_handlers_{};
+    std::mutex command_handlers_mutex_{};
+    std::map<uint32_t, std::function<void(std::error_code, retry_reason, io::mcbp_message&&)>> command_handlers_{};
+    std::vector<std::function<void(const configuration&)>> config_listeners_{};
+    std::function<void(io::retry_reason)> on_stop_handler_{};
 
     bool bootstrapped_{ false };
     std::atomic_bool stopped_{ false };
     bool authenticated_{ false };
     bool bucket_selected_{ false };
     bool supports_gcccp_{ true };
+    bool retry_bootstrap_on_bucket_not_found_{ false };
 
     std::atomic<std::uint32_t> opaque_{ 0 };
 
@@ -1031,16 +1336,21 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     std::mutex pending_buffer_mutex_{};
     std::mutex writing_buffer_mutex_{};
     std::string bootstrap_hostname_{};
+    std::string bootstrap_port_{};
     asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
     std::string endpoint_address_{};     // cached string with endpoint address
+    asio::ip::tcp::endpoint local_endpoint_{};
+    std::string local_endpoint_address_{};
     asio::ip::tcp::resolver::results_type endpoints_;
     std::vector<protocol::hello_feature> supported_features_;
     std::optional<configuration> config_;
-    std::optional<error_map> errmap_;
+    std::optional<error_map> error_map_;
     collection_cache collection_cache_;
 
     std::atomic_bool reading_{ false };
 
     std::string log_prefix_{};
+    std::chrono::time_point<std::chrono::steady_clock> last_active_{};
+    diag::endpoint_state state_{ diag::endpoint_state::disconnected };
 };
 } // namespace couchbase::io