couchbase 3.0.0.beta.1 → 3.0.0

data/ext/couchbase/errors.hxx

@@ -247,7 +247,9 @@ enum class analytics_errc {
 };
 
 /// Errors related to Search service (CBFT)
-enum class search_errc {};
+enum class search_errc {
+    index_not_ready = 400,
+};
 
 /// Errors related to Views service (CAPI)
 enum class view_errc {
@@ -465,8 +467,12 @@ struct search_error_category : std::error_category {
         return "search";
     }
 
-    [[nodiscard]] std::string message(int) const noexcept override
+    [[nodiscard]] std::string message(int ev) const noexcept override
     {
+        switch (static_cast<search_errc>(ev)) {
+            case search_errc::index_not_ready:
+                return "index_not_ready";
+        }
         return "FIXME: unknown error code in search category (recompile with newer library)";
     }
 };

data/ext/couchbase/io/dns_client.hxx

@@ -81,18 +81,17 @@ class dns_client
                                                                                     std::size_t /* bytes_transferred */) mutable {
                   if (ec1 == asio::error::operation_aborted) {
                       self->deadline_.cancel();
-                      return handler({ std::make_error_code(error::common_errc::ambiguous_timeout) });
+                      return handler({ std::make_error_code(error::common_errc::unambiguous_timeout) });
                   }
                   if (ec1) {
                       self->deadline_.cancel();
                       return handler({ ec1 });
                   }
 
-                  asio::ip::udp::endpoint sender_endpoint;
                   self->recv_buf_.resize(512);
                   self->udp_.async_receive_from(
                     asio::buffer(self->recv_buf_),
-                    sender_endpoint,
+                    self->udp_sender_,
                     [self, handler = std::forward<Handler>(handler)](std::error_code ec2, std::size_t bytes_transferred) mutable {
                         self->deadline_.cancel();
                         if (ec2) {
@@ -149,7 +148,7 @@ class dns_client
                         if (ec2) {
                             self->deadline_.cancel();
                             if (ec2 == asio::error::operation_aborted) {
-                                ec2 = std::make_error_code(error::common_errc::ambiguous_timeout);
+                                ec2 = std::make_error_code(error::common_errc::unambiguous_timeout);
                             }
                             return handler({ ec2 });
                         }
@@ -189,14 +188,15 @@ class dns_client
 
         asio::steady_timer deadline_;
         asio::ip::udp::socket udp_;
+        asio::ip::udp::endpoint udp_sender_{};
         asio::ip::tcp::socket tcp_;
 
         asio::ip::address address_;
         std::uint16_t port_;
 
-        std::vector<uint8_t> send_buf_;
+        std::vector<uint8_t> send_buf_{};
         std::uint16_t recv_buf_size_{ 0 };
-        std::vector<uint8_t> recv_buf_;
+        std::vector<uint8_t> recv_buf_{};
     };
 
     explicit dns_client(asio::io_context& ctx)

data/ext/couchbase/io/http_command.hxx

@@ -45,7 +45,7 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
         request.encode_to(encoded);
         encoded.headers["client-context-id"] = request.client_context_id;
         auto log_prefix = session->log_prefix();
-        spdlog::debug("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms",
+        spdlog::trace("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms",
                       log_prefix,
                       encoded.type,
                       encoded.method,
@@ -65,7 +65,7 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
                                        std::error_code ec, io::http_response&& msg) mutable {
                                          self->deadline.cancel();
                                          encoded_response_type resp(msg);
-                                         spdlog::debug("{} HTTP response: {}, client_context_id={}, status={}",
+                                         spdlog::trace("{} HTTP response: {}, client_context_id={}, status={}",
                                                        log_prefix,
                                                        self->request.type,
                                                        self->request.client_context_id,

data/ext/couchbase/io/http_session.hxx

@@ -43,8 +43,7 @@ class http_session : public std::enable_shared_from_this<http_session>
   public:
     http_session(const std::string& client_id,
                  asio::io_context& ctx,
-                 const std::string& username,
-                 const std::string& password,
+                 const cluster_credentials& credentials,
                  const std::string& hostname,
                  const std::string& service)
       : client_id_(client_id)
@@ -53,8 +52,7 @@ class http_session : public std::enable_shared_from_this<http_session>
       , resolver_(ctx_)
       , stream_(std::make_unique<plain_stream_impl>(ctx_))
       , deadline_timer_(ctx_)
-      , username_(username)
-      , password_(password)
+      , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
       , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
@@ -72,8 +70,7 @@ class http_session : public std::enable_shared_from_this<http_session>
     http_session(const std::string& client_id,
                  asio::io_context& ctx,
                  asio::ssl::context& tls,
-                 const std::string& username,
-                 const std::string& password,
+                 const cluster_credentials& credentials,
                  const std::string& hostname,
                  const std::string& service)
       : client_id_(client_id)
@@ -82,8 +79,7 @@ class http_session : public std::enable_shared_from_this<http_session>
       , resolver_(ctx_)
       , stream_(std::make_unique<tls_stream_impl>(ctx_, tls))
       , deadline_timer_(ctx_)
-      , username_(username)
-      , password_(password)
+      , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
       , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
@@ -191,7 +187,8 @@ class http_session : public std::enable_shared_from_this<http_session>
             return;
         }
         request.headers["user-agent"] = user_agent_;
-        request.headers["authorization"] = fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", username_, password_)));
+        request.headers["authorization"] =
+          fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", credentials_.username, credentials_.password)));
         write(fmt::format("{} {} HTTP/1.1\r\nhost: {}:{}\r\n", request.method, request.path, hostname_, service_));
         if (!request.body.empty()) {
             request.headers["content-length"] = std::to_string(request.body.size());
@@ -335,8 +332,7 @@ class http_session : public std::enable_shared_from_this<http_session>
     std::unique_ptr<stream_impl> stream_;
     asio::steady_timer deadline_timer_;
 
-    std::string username_;
-    std::string password_;
+    cluster_credentials credentials_;
     std::string hostname_;
     std::string service_;
     std::string user_agent_;

data/ext/couchbase/io/http_session_manager.hxx

@@ -48,7 +48,7 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
         }
     }
 
-    std::shared_ptr<http_session> check_out(service_type type, const std::string& username, const std::string& password)
+    std::shared_ptr<http_session> check_out(service_type type, const couchbase::cluster_credentials& credentials)
     {
         std::scoped_lock lock(sessions_mutex_);
         if (idle_sessions_[type].empty()) {
@@ -61,9 +61,9 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
             config_.nodes.size();
             std::shared_ptr<http_session> session;
             if (options_.enable_tls) {
-                session = std::make_shared<http_session>(client_id_, ctx_, tls_, username, password, hostname, std::to_string(port));
+                session = std::make_shared<http_session>(client_id_, ctx_, tls_, credentials, hostname, std::to_string(port));
             } else {
-                session = std::make_shared<http_session>(client_id_, ctx_, username, password, hostname, std::to_string(port));
+                session = std::make_shared<http_session>(client_id_, ctx_, credentials, hostname, std::to_string(port));
             }
             session->start();
             session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {

data/ext/couchbase/io/mcbp_command.hxx

@@ -18,6 +18,8 @@
 #pragma once
 
 #include <io/mcbp_session.hxx>
+#include <io/retry_orchestrator.hxx>
+
 #include <protocol/cmd_get_collection_id.hxx>
 #include <functional>
 #include <utility>
@@ -27,8 +29,8 @@ namespace couchbase::operations
 
 using mcbp_command_handler = std::function<void(std::error_code, std::optional<io::mcbp_message>)>;
 
-template<typename Request>
-struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>> {
+template<typename Manager, typename Request>
+struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager, Request>> {
     using encoded_request_type = typename Request::encoded_request_type;
     using encoded_response_type = typename Request::encoded_response_type;
     asio::steady_timer deadline;
@@ -38,11 +40,13 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
     std::optional<std::uint32_t> opaque_{};
     std::shared_ptr<io::mcbp_session> session_{};
     mcbp_command_handler handler_{};
+    std::shared_ptr<Manager> manager_{};
 
-    mcbp_command(asio::io_context& ctx, Request req)
+    mcbp_command(asio::io_context& ctx, std::shared_ptr<Manager> manager, Request req)
       : deadline(ctx)
       , retry_backoff(ctx)
       , request(req)
+      , manager_(manager)
     {
     }
 
@@ -54,16 +58,21 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
             if (ec == asio::error::operation_aborted) {
                 return;
             }
-            self->cancel();
+            self->cancel(io::retry_reason::do_not_retry);
         });
     }
 
-    void cancel()
+    void cancel(io::retry_reason reason)
     {
         if (opaque_ && session_) {
-            session_->cancel(opaque_.value(), asio::error::operation_aborted);
+            if (session_->cancel(opaque_.value(), asio::error::operation_aborted, reason)) {
+                handler_ = nullptr;
+            }
         }
-        handler_ = nullptr;
+        invoke_handler(std::make_error_code(request.retries.idempotent ? error::common_errc::unambiguous_timeout
+                                                                       : error::common_errc::ambiguous_timeout));
+        retry_backoff.cancel();
+        deadline.cancel();
     }
 
     void invoke_handler(std::error_code ec, std::optional<io::mcbp_message> msg = {})
@@ -79,26 +88,27 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
         protocol::client_request<protocol::get_collection_id_request_body> req;
         req.opaque(session_->next_opaque());
         req.body().collection_path(request.id.collection);
-        session_->write_and_subscribe(req.opaque(),
-                                      req.data(session_->supports_feature(protocol::hello_feature::snappy)),
-                                      [self = this->shared_from_this()](std::error_code ec, io::mcbp_message&& msg) mutable {
-                                          if (ec == asio::error::operation_aborted) {
-                                              return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
-                                          }
-                                          if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
-                                              if (self->request.id.collection_uid) {
-                                                  return self->handle_unknown_collection();
-                                              }
-                                              return self->invoke_handler(ec);
-                                          }
-                                          if (ec) {
-                                              return self->invoke_handler(ec);
-                                          }
-                                          protocol::client_response<protocol::get_collection_id_response_body> resp(msg);
-                                          self->session_->update_collection_uid(self->request.id.collection, resp.body().collection_uid());
-                                          self->request.id.collection_uid = resp.body().collection_uid();
-                                          return self->send();
-                                      });
+        session_->write_and_subscribe(
+          req.opaque(),
+          req.data(session_->supports_feature(protocol::hello_feature::snappy)),
+          [self = this->shared_from_this()](std::error_code ec, io::retry_reason /* reason */, io::mcbp_message&& msg) mutable {
+              if (ec == asio::error::operation_aborted) {
+                  return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
+              }
+              if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
+                  if (self->request.id.collection_uid) {
+                      return self->handle_unknown_collection();
+                  }
+                  return self->invoke_handler(ec);
+              }
+              if (ec) {
+                  return self->invoke_handler(ec);
+              }
+              protocol::client_response<protocol::get_collection_id_response_body> resp(msg);
+              self->session_->update_collection_uid(self->request.id.collection, resp.body().collection_uid());
+              self->request.id.collection_uid = resp.body().collection_uid();
+              return self->send();
+          });
     }
 
     void handle_unknown_collection()
@@ -112,7 +122,8 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
                       request.id.key,
                       std::chrono::duration_cast<std::chrono::milliseconds>(time_left).count());
         if (time_left < backoff) {
-            return invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
+            return invoke_handler(std::make_error_code(request.retries.idempotent ? error::common_errc::unambiguous_timeout
+                                                                                  : error::common_errc::ambiguous_timeout));
         }
         retry_backoff.expires_after(backoff);
         retry_backoff.async_wait([self = this->shared_from_this()](std::error_code ec) mutable {
@@ -149,22 +160,68 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
         }
         request.encode_to(encoded);
 
-        session_->write_and_subscribe(request.opaque,
-                                      encoded.data(session_->supports_feature(protocol::hello_feature::snappy)),
-                                      [self = this->shared_from_this()](std::error_code ec, io::mcbp_message&& msg) mutable {
-                                          self->retry_backoff.cancel();
-                                          if (ec == asio::error::operation_aborted) {
-                                              return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
-                                          }
-                                          if (ec == std::make_error_code(error::common_errc::request_canceled)) {
-                                              return self->invoke_handler(ec);
-                                          }
-                                          if (msg.header.status() == static_cast<std::uint16_t>(protocol::status::unknown_collection)) {
-                                              return self->handle_unknown_collection();
-                                          }
-                                          self->deadline.cancel();
-                                          self->invoke_handler(ec, msg);
-                                      });
+        session_->write_and_subscribe(
+          request.opaque,
+          encoded.data(session_->supports_feature(protocol::hello_feature::snappy)),
+          [self = this->shared_from_this()](std::error_code ec, io::retry_reason reason, io::mcbp_message&& msg) mutable {
+              self->retry_backoff.cancel();
+              if (ec == asio::error::operation_aborted) {
+                  return self->invoke_handler(std::make_error_code(
+                    self->request.retries.idempotent ? error::common_errc::unambiguous_timeout : error::common_errc::ambiguous_timeout));
+              }
+              if (ec == std::make_error_code(error::common_errc::request_canceled)) {
+                  if (reason == io::retry_reason::do_not_retry) {
+                      return self->invoke_handler(ec);
+                  }
+                  return io::retry_orchestrator::maybe_retry(self->manager_, self, reason, ec);
+              }
+              protocol::status status = protocol::status::invalid;
+              std::optional<error_map::error_info> error_code{};
+              if (protocol::is_valid_status(msg.header.status())) {
+                  status = static_cast<protocol::status>(msg.header.status());
+              } else {
+                  error_code = self->session_->decode_error_code(msg.header.status());
+              }
+              if (status == protocol::status::not_my_vbucket) {
+                  self->session_->handle_not_my_vbucket(std::move(msg));
+                  return io::retry_orchestrator::maybe_retry(self->manager_, self, io::retry_reason::kv_not_my_vbucket, ec);
+              }
+              if (status == protocol::status::unknown_collection) {
+                  return self->handle_unknown_collection();
+              }
+              if (error_code && error_code.value().has_retry_attribute()) {
+                  reason = io::retry_reason::kv_error_map_retry_indicated;
+              } else {
+                  switch (status) {
+                      case protocol::status::locked:
+                          if (encoded_request_type::body_type::opcode != protocol::client_opcode::unlock) {
+                              /**
+                               * special case for unlock command, when it should not be retried, because it does not make sense
+                               * (someone else unlocked the document)
+                               */
+                              reason = io::retry_reason::kv_locked;
+                          }
+                          break;
+                      case protocol::status::temporary_failure:
+                          reason = io::retry_reason::kv_temporary_failure;
+                          break;
+                      case protocol::status::sync_write_in_progress:
+                          reason = io::retry_reason::kv_sync_write_in_progress;
+                          break;
+                      case protocol::status::sync_write_re_commit_in_progress:
+                          reason = io::retry_reason::kv_sync_write_re_commit_in_progress;
+                          break;
+                      default:
+                          break;
+                  }
+              }
+              if (reason == io::retry_reason::do_not_retry) {
+                  self->deadline.cancel();
+                  self->invoke_handler(ec, msg);
+              } else {
+                  io::retry_orchestrator::maybe_retry(self->manager_, self, reason, ec);
+              }
+          });
     }
 
     void send_to(std::shared_ptr<io::mcbp_session> session)

data/ext/couchbase/io/mcbp_session.hxx

@@ -28,6 +28,7 @@
 #include <io/mcbp_message.hxx>
 #include <io/mcbp_parser.hxx>
 #include <io/streams.hxx>
+#include <io/retry_orchestrator.hxx>
 
 #include <timeout_defaults.hxx>
 
@@ -120,8 +121,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
         explicit bootstrap_handler(std::shared_ptr<mcbp_session> session)
           : session_(session)
-          , sasl_([origin = session_->origin_]() -> std::string { return origin.get_username(); },
-                  [origin = session_->origin_]() -> std::string { return origin.get_password(); },
+          , sasl_([origin = session_->origin_]() -> std::string { return origin.username(); },
+                  [origin = session_->origin_]() -> std::string { return origin.password(); },
                   { "SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1", "PLAIN" })
         {
             tao::json::value user_agent{
@@ -139,18 +140,20 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           fmt::join(hello_req.body().features(), ", "));
             session_->write(hello_req.data());
 
-            protocol::client_request<protocol::sasl_list_mechs_request_body> list_req;
-            list_req.opaque(session_->next_opaque());
-            session_->write(list_req.data());
-
-            protocol::client_request<protocol::sasl_auth_request_body> auth_req;
-            sasl::error sasl_code;
-            std::string_view sasl_payload;
-            std::tie(sasl_code, sasl_payload) = sasl_.start();
-            auth_req.opaque(session_->next_opaque());
-            auth_req.body().mechanism(sasl_.get_name());
-            auth_req.body().sasl_data(sasl_payload);
-            session_->write(auth_req.data());
+            if (!session->origin_.credentials().uses_certificate()) {
+                protocol::client_request<protocol::sasl_list_mechs_request_body> list_req;
+                list_req.opaque(session_->next_opaque());
+                session_->write(list_req.data());
+
+                protocol::client_request<protocol::sasl_auth_request_body> auth_req;
+                sasl::error sasl_code;
+                std::string_view sasl_payload;
+                std::tie(sasl_code, sasl_payload) = sasl_.start();
+                auth_req.opaque(session_->next_opaque());
+                auth_req.body().mechanism(sasl_.get_name());
+                auth_req.body().sasl_data(sasl_payload);
+                session_->write(auth_req.data());
+            }
 
             session_->flush();
         }
@@ -194,6 +197,10 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                     if (resp.status() == protocol::status::success) {
                         session_->supported_features_ = resp.body().supported_features();
                         spdlog::debug("{} supported_features=[{}]", session_->log_prefix_, fmt::join(session_->supported_features_, ", "));
+                        if (session_->origin_.credentials().uses_certificate()) {
+                            spdlog::debug("{} skip SASL authentication, because TLS certificate was specified", session_->log_prefix_);
+                            return auth_success();
+                        }
                     } else {
                         spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
                         return complete(std::make_error_code(error::network_errc::handshake_failure));
@@ -246,7 +253,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 case protocol::client_opcode::get_error_map: {
                     protocol::client_response<protocol::get_error_map_response_body> resp(msg);
                     if (resp.status() == protocol::status::success) {
-                        session_->errmap_.emplace(resp.body().errmap());
+                        session_->error_map_.emplace(resp.body().errmap());
                     } else {
                         spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
                                      session_->log_prefix_,
@@ -369,16 +376,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                             auto handler = session_->command_handlers_.find(opaque);
                             if (handler != session_->command_handlers_.end()) {
                                 auto ec = session_->map_status_code(opcode, status);
-                                spdlog::debug("{} MCBP invoke operation handler, opaque={}, status={}, ec={}",
+                                spdlog::trace("{} MCBP invoke operation handler: opaque={}, status={}, ec={}",
                                               session_->log_prefix_,
                                               opaque,
-                                              status,
+                                              protocol::status_to_string(status),
                                               ec.message());
                                 auto fun = handler->second;
                                 session_->command_handlers_.erase(handler);
-                                fun(ec, std::move(msg));
+                                fun(ec, retry_reason::do_not_retry, std::move(msg));
                             } else {
-                                spdlog::debug("{} unexpected orphan response opcode={}, opaque={}",
+                                spdlog::debug("{} unexpected orphan response: opcode={}, opaque={}",
                                               session_->log_prefix_,
                                               msg.header.opcode,
                                               msg.header.opaque);
@@ -472,7 +479,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     ~mcbp_session()
     {
-        stop();
+        stop(retry_reason::do_not_retry);
     }
 
     [[nodiscard]] const std::string& log_prefix() const
@@ -480,8 +487,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return log_prefix_;
     }
 
-    void bootstrap(std::function<void(std::error_code, configuration)>&& handler)
+    void bootstrap(std::function<void(std::error_code, configuration)>&& handler, bool retry_on_bucket_not_found = false)
     {
+        retry_bootstrap_on_bucket_not_found_ = retry_on_bucket_not_found;
         bootstrap_handler_ = std::move(handler);
         bootstrap_deadline_.expires_after(timeout_defaults::bootstrap_timeout);
         bootstrap_deadline_.async_wait([self = shared_from_this()](std::error_code ec) {
@@ -491,7 +499,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             spdlog::warn("{} unable to bootstrap in time", self->log_prefix_);
             self->bootstrap_handler_(std::make_error_code(error::common_errc::unambiguous_timeout), {});
             self->bootstrap_handler_ = nullptr;
-            self->stop();
+            self->stop(retry_reason::socket_closed_while_in_flight);
         });
         initiate_bootstrap();
     }
@@ -514,13 +522,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             });
             return;
         }
-        std::string service;
-        std::tie(bootstrap_hostname_, service) = origin_.next_address();
-        log_prefix_ = fmt::format(
-          "[{}/{}/{}/{}] <{}:{}>", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"), bootstrap_hostname_, service);
+        std::tie(bootstrap_hostname_, bootstrap_port_) = origin_.next_address();
+        log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}:{}>",
+                                  stream_->log_prefix(),
+                                  client_id_,
+                                  id_,
+                                  bucket_name_.value_or("-"),
+                                  bootstrap_hostname_,
+                                  bootstrap_port_);
         spdlog::debug("{} attempt to establish MCBP connection", log_prefix_);
         resolver_.async_resolve(bootstrap_hostname_,
-                                service,
+                                bootstrap_port_,
                                 std::bind(&mcbp_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
     }
 
@@ -529,11 +541,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return id_;
     }
 
-    void stop()
+    void on_stop(std::function<void(io::retry_reason)> handler)
+    {
+        on_stop_handler_ = std::move(handler);
+    }
+
+    void stop(retry_reason reason)
     {
         if (stopped_) {
             return;
         }
+        spdlog::debug("{} stop MCBP connection, reason={}", log_prefix_, reason);
         stopped_ = true;
         bootstrap_deadline_.cancel();
         connection_deadline_.cancel();
@@ -552,9 +570,14 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         for (auto& handler : command_handlers_) {
             spdlog::debug("{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
-            handler.second(ec, {});
+            handler.second(ec, reason, {});
         }
         command_handlers_.clear();
+        config_listeners_.clear();
+        if (on_stop_handler_) {
+            on_stop_handler_(reason);
+        }
+        on_stop_handler_ = nullptr;
     }
 
     void write(const std::vector<uint8_t>& buf)
@@ -564,7 +587,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         std::uint32_t opaque{ 0 };
         std::memcpy(&opaque, buf.data() + 12, sizeof(opaque));
-        spdlog::debug("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
+        spdlog::trace("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
         SPDLOG_TRACE("{} MCBP send, opaque={}{:a}", log_prefix_, opaque, spdlog::to_hex(data));
         std::scoped_lock lock(output_buffer_mutex_);
         output_buffer_.push_back(buf);
@@ -589,33 +612,36 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void write_and_subscribe(uint32_t opaque,
                              std::vector<std::uint8_t>& data,
-                             std::function<void(std::error_code, io::mcbp_message&&)> handler)
+                             std::function<void(std::error_code, retry_reason, io::mcbp_message&&)> handler)
     {
         if (stopped_) {
-            spdlog::warn("{} MCBP cancel operation, while trying to write to closed session opaque={}", log_prefix_, opaque);
-            handler(std::make_error_code(error::common_errc::request_canceled), {});
+            spdlog::warn("{} MCBP cancel operation, while trying to write to closed session, opaque={}", log_prefix_, opaque);
+            handler(std::make_error_code(error::common_errc::request_canceled), retry_reason::socket_closed_while_in_flight, {});
             return;
         }
         command_handlers_.emplace(opaque, std::move(handler));
         if (bootstrapped_ && stream_->is_open()) {
             write_and_flush(data);
         } else {
+            spdlog::debug("{} the stream is not ready yet, put the message into pending buffer, opaque={}", log_prefix_, opaque);
             std::scoped_lock lock(pending_buffer_mutex_);
             pending_buffer_.push_back(data);
         }
     }
 
-    void cancel(uint32_t opaque, std::error_code ec)
+    [[nodiscard]] bool cancel(uint32_t opaque, std::error_code ec, retry_reason reason)
     {
         if (stopped_) {
-            return;
+            return false;
         }
         auto handler = command_handlers_.find(opaque);
         if (handler != command_handlers_.end()) {
             spdlog::debug("{} MCBP cancel operation, opaque={}, ec={}", log_prefix_, opaque, ec.message());
-            handler->second(ec, {});
+            handler->second(ec, reason, {});
             command_handlers_.erase(handler);
+            return true;
         }
+        return false;
     }
 
     [[nodiscard]] bool supports_feature(protocol::hello_feature feature)
@@ -654,6 +680,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return bootstrap_hostname_;
     }
 
+    [[nodiscard]] const std::string& bootstrap_port() const
+    {
+        return bootstrap_port_;
+    }
+
     [[nodiscard]] uint32_t next_opaque()
     {
         return ++opaque_;
@@ -708,7 +739,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 return std::make_error_code(error::common_errc::internal_server_failure);
 
             case protocol::status::busy:
-            case protocol::status::temp_failure:
+            case protocol::status::temporary_failure:
             case protocol::status::no_memory:
             case protocol::status::not_initialized:
                 return std::make_error_code(error::common_errc::temporary_failure);
@@ -793,10 +824,26 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 break;
         }
         // FIXME: use error map here
-        spdlog::warn("{} unknown status code: {} (opcode={})", log_prefix_, status, opcode);
+        spdlog::warn("{} unknown status code: {} (opcode={})", log_prefix_, protocol::status_to_string(status), opcode);
         return std::make_error_code(error::network_errc::protocol_error);
     }
 
+    std::optional<error_map::error_info> decode_error_code(std::uint16_t code)
+    {
+        if (error_map_) {
+            auto info = error_map_->errors.find(code);
+            if (info != error_map_->errors.end()) {
+                return info->second;
+            }
+        }
+        return {};
+    }
+
+    void on_configuration_update(std::function<void(const configuration&)> handler)
+    {
+        config_listeners_.emplace_back(std::move(handler));
+    }
+
     void update_configuration(configuration&& config)
     {
         if (stopped_) {
@@ -804,12 +851,45 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         if (!config_ || config.rev > config_->rev) {
             for (auto& node : config.nodes) {
-                if (node.this_node && node.hostname.empty()) {
-                    node.hostname = endpoint_address_;
+                if (node.hostname.empty()) {
+                    node.hostname = bootstrap_hostname_;
                 }
             }
             config_.emplace(config);
             spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
+            for (auto& listener : config_listeners_) {
+                listener(*config_);
+            }
+        }
+    }
+
+    void handle_not_my_vbucket(io::mcbp_message&& msg)
+    {
+        if (stopped_) {
+            return;
+        }
+        Expects(msg.header.magic == static_cast<std::uint8_t>(protocol::magic::alt_client_response) ||
+                msg.header.magic == static_cast<std::uint8_t>(protocol::magic::client_response));
+        if (protocol::has_json_datatype(msg.header.datatype)) {
+            auto magic = static_cast<protocol::magic>(msg.header.magic);
+            uint8_t extras_size = msg.header.extlen;
+            uint8_t framing_extras_size = 0;
+            uint16_t key_size = htons(msg.header.keylen);
+            if (magic == protocol::magic::alt_client_response) {
+                framing_extras_size = static_cast<std::uint8_t>(msg.header.keylen >> 8U);
+                key_size = msg.header.keylen & 0xffU;
+            }
+
+            std::vector<uint8_t>::difference_type offset = framing_extras_size + key_size + extras_size;
+            if (ntohl(msg.header.bodylen) - offset > 0) {
+                auto config = protocol::parse_config(msg.body.begin() + offset, msg.body.end());
+                spdlog::debug("{} received not_my_vbucket status for {}, opaque={} with config rev={} in the payload",
+                              log_prefix_,
+                              static_cast<protocol::client_opcode>(msg.header.opcode),
+                              msg.header.opaque,
+                              config.rev);
+                update_configuration(std::move(config));
+            }
         }
     }
 
@@ -829,13 +909,18 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
   private:
     void invoke_bootstrap_handler(std::error_code ec)
     {
+        if (retry_bootstrap_on_bucket_not_found_ && ec == std::make_error_code(error::common_errc::bucket_not_found)) {
+            spdlog::debug(R"({} server returned {}, it must be transient condition, retrying)", log_prefix_, ec.message());
+            return initiate_bootstrap();
+        }
+
         if (!bootstrapped_ && bootstrap_handler_) {
             bootstrap_deadline_.cancel();
             bootstrap_handler_(ec, config_.value_or(configuration{}));
             bootstrap_handler_ = nullptr;
         }
         if (ec) {
-            return stop();
+            return stop(retry_reason::node_not_available);
         }
         bootstrapped_ = true;
         handler_ = std::make_unique<normal_handler>(shared_from_this());
@@ -934,8 +1019,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                   return;
               }
               if (ec) {
-                  spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
-                  return self->stop();
+                  spdlog::error("{} IO error while reading from the socket: {}, pending_handlers={}",
+                                self->log_prefix_,
+                                ec.message(),
+                                self->command_handlers_.size());
+                  return self->stop(retry_reason::socket_closed_while_in_flight);
               }
               self->parser_.feed(self->input_buffer_.data(), self->input_buffer_.data() + ssize_t(bytes_transferred));
 
@@ -943,7 +1031,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                   mcbp_message msg{};
                   switch (self->parser_.next(msg)) {
                       case mcbp_parser::ok:
-                          spdlog::debug(
+                          spdlog::trace(
                             "{} MCBP recv, opaque={}, {:n}", self->log_prefix_, msg.header.opaque, spdlog::to_hex(msg.header_data()));
                           SPDLOG_TRACE("{} MCBP recv, opaque={}{:a}{:a}",
                                        self->log_prefix_,
@@ -962,7 +1050,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                           }
                           return;
                       case mcbp_parser::failure:
-                          return self->stop();
+                          return self->stop(retry_reason::kv_temporary_failure);
                   }
               }
           });
@@ -988,8 +1076,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                 return;
             }
             if (ec) {
-                spdlog::error("{} IO error while writing to the socket: {}", self->log_prefix_, ec.message());
-                return self->stop();
+                spdlog::error("{} IO error while writing to the socket: {}, pending_handlers={}",
+                              self->log_prefix_,
+                              ec.message(),
+                              self->command_handlers_.size());
+                return self->stop(retry_reason::socket_closed_while_in_flight);
             }
             {
                 std::scoped_lock inner_lock(self->writing_buffer_mutex_);
@@ -1013,13 +1104,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     mcbp_parser parser_;
     std::unique_ptr<message_handler> handler_;
     std::function<void(std::error_code, const configuration&)> bootstrap_handler_{};
-    std::map<uint32_t, std::function<void(std::error_code, io::mcbp_message&&)>> command_handlers_{};
+    std::map<uint32_t, std::function<void(std::error_code, retry_reason, io::mcbp_message&&)>> command_handlers_{};
+    std::vector<std::function<void(const configuration&)>> config_listeners_{};
+    std::function<void(io::retry_reason)> on_stop_handler_{};
 
     bool bootstrapped_{ false };
     std::atomic_bool stopped_{ false };
     bool authenticated_{ false };
     bool bucket_selected_{ false };
     bool supports_gcccp_{ true };
+    bool retry_bootstrap_on_bucket_not_found_{ false };
 
     std::atomic<std::uint32_t> opaque_{ 0 };
 
@@ -1031,12 +1125,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     std::mutex pending_buffer_mutex_{};
     std::mutex writing_buffer_mutex_{};
     std::string bootstrap_hostname_{};
+    std::string bootstrap_port_{};
     asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
     std::string endpoint_address_{};     // cached string with endpoint address
     asio::ip::tcp::resolver::results_type endpoints_;
     std::vector<protocol::hello_feature> supported_features_;
     std::optional<configuration> config_;
-    std::optional<error_map> errmap_;
+    std::optional<error_map> error_map_;
     collection_cache collection_cache_;
 
     std::atomic_bool reading_{ false };