couchbase 3.0.0.alpha.5 → 3.0.0.beta.1

data/ext/couchbase/io/mcbp_message.hxx

@@ -41,6 +41,11 @@ struct binary_header {
     std::uint32_t bodylen;
     std::uint32_t opaque;
     std::uint64_t cas;
+
+    std::uint16_t status()
+    {
+        return htons(specific);
+    }
 };
 
 struct mcbp_message {

data/ext/couchbase/io/mcbp_session.hxx

@@ -27,6 +27,7 @@
 
 #include <io/mcbp_message.hxx>
 #include <io/mcbp_parser.hxx>
+#include <io/streams.hxx>
 
 #include <timeout_defaults.hxx>
 
@@ -48,6 +49,7 @@
 
 #include <spdlog/fmt/bin_to_hex.h>
 
+#include <origin.hxx>
 #include <errors.hxx>
 #include <version.hxx>
 
@@ -118,8 +120,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
         explicit bootstrap_handler(std::shared_ptr<mcbp_session> session)
           : session_(session)
-          , sasl_([this]() -> std::string { return session_->username_; },
-                  [this]() -> std::string { return session_->password_; },
+          , sasl_([origin = session_->origin_]() -> std::string { return origin.get_username(); },
+                  [origin = session_->origin_]() -> std::string { return origin.get_password(); },
                   { "SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1", "PLAIN" })
         {
             tao::json::value user_agent{
@@ -155,10 +157,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
         void complete(std::error_code ec)
         {
+            stopped_ = true;
             session_->invoke_bootstrap_handler(ec);
-            if (!ec) {
-                session_->handler_ = std::make_unique<normal_handler>(session_);
-            }
         }
 
         void auth_success()
@@ -282,7 +282,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         spdlog::warn("{} this server does not support GCCCP, open bucket before making any cluster-level command",
                                      session_->log_prefix_);
                         session_->update_configuration(
-                          make_blank_configuration(session_->endpoint_.address().to_string(), session_->endpoint_.port(), 0));
+                          make_blank_configuration(session_->endpoint_address_, session_->endpoint_.port(), 0));
                         complete({});
                     } else {
                         spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
@@ -374,8 +374,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                                               opaque,
                                               status,
                                               ec.message());
-                                handler->second(ec, std::move(msg));
+                                auto fun = handler->second;
                                 session_->command_handlers_.erase(handler);
+                                fun(ec, std::move(msg));
                             } else {
                                 spdlog::debug("{} unexpected orphan response opcode={}, opaque={}",
                                               session_->log_prefix_,
@@ -415,10 +416,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
         void fetch_config(std::error_code ec)
         {
-            if (ec == asio::error::operation_aborted) {
-                return;
-            }
-            if (stopped_ || !session_) {
+            if (ec == asio::error::operation_aborted || stopped_ || !session_) {
                 return;
             }
             protocol::client_request<protocol::get_cluster_config_request_body> req;
@@ -430,21 +428,46 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     };
 
   public:
+    mcbp_session() = delete;
     mcbp_session(const std::string& client_id,
                  asio::io_context& ctx,
+                 const couchbase::origin& origin,
                  std::optional<std::string> bucket_name = {},
                  std::vector<protocol::hello_feature> known_features = {})
       : client_id_(client_id)
       , id_(uuid::to_string(uuid::random()))
       , ctx_(ctx)
       , resolver_(ctx_)
-      , strand_(asio::make_strand(ctx_))
-      , socket_(strand_)
-      , deadline_timer_(ctx_)
+      , stream_(std::make_unique<plain_stream_impl>(ctx_))
+      , bootstrap_deadline_(ctx_)
+      , connection_deadline_(ctx_)
+      , retry_backoff_(ctx_)
+      , origin_(origin)
       , bucket_name_(std::move(bucket_name))
       , supported_features_(known_features)
     {
-        log_prefix_ = fmt::format("[{}/{}/{}]", client_id_, id_, bucket_name_.value_or("-"));
+        log_prefix_ = fmt::format("[{}/{}/{}/{}]", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"));
+    }
+
+    mcbp_session(const std::string& client_id,
+                 asio::io_context& ctx,
+                 asio::ssl::context& tls,
+                 const couchbase::origin& origin,
+                 std::optional<std::string> bucket_name = {},
+                 std::vector<protocol::hello_feature> known_features = {})
+      : client_id_(client_id)
+      , id_(uuid::to_string(uuid::random()))
+      , ctx_(ctx)
+      , resolver_(ctx_)
+      , stream_(std::make_unique<tls_stream_impl>(ctx_, tls))
+      , bootstrap_deadline_(ctx_)
+      , connection_deadline_(ctx_)
+      , retry_backoff_(ctx_)
+      , origin_(origin)
+      , bucket_name_(std::move(bucket_name))
+      , supported_features_(known_features)
+    {
+        log_prefix_ = fmt::format("[{}/{}/{}/{}]", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"));
     }
 
     ~mcbp_session()
@@ -452,17 +475,53 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         stop();
     }
 
-    void bootstrap(const std::string& hostname,
-                   const std::string& service,
-                   const std::string& username,
-                   const std::string& password,
-                   std::function<void(std::error_code, configuration)>&& handler)
+    [[nodiscard]] const std::string& log_prefix() const
+    {
+        return log_prefix_;
+    }
+
+    void bootstrap(std::function<void(std::error_code, configuration)>&& handler)
     {
-        username_ = username;
-        password_ = password;
         bootstrap_handler_ = std::move(handler);
-        resolver_.async_resolve(
-          hostname, service, std::bind(&mcbp_session::on_resolve, this, std::placeholders::_1, std::placeholders::_2));
+        bootstrap_deadline_.expires_after(timeout_defaults::bootstrap_timeout);
+        bootstrap_deadline_.async_wait([self = shared_from_this()](std::error_code ec) {
+            if (ec == asio::error::operation_aborted || self->stopped_) {
+                return;
+            }
+            spdlog::warn("{} unable to bootstrap in time", self->log_prefix_);
+            self->bootstrap_handler_(std::make_error_code(error::common_errc::unambiguous_timeout), {});
+            self->bootstrap_handler_ = nullptr;
+            self->stop();
+        });
+        initiate_bootstrap();
+    }
+
+    void initiate_bootstrap()
+    {
+        if (stopped_) {
+            return;
+        }
+        if (origin_.exhausted()) {
+            auto backoff = std::chrono::milliseconds(500);
+            spdlog::debug("{} reached the end of list of bootstrap nodes, waiting for {}ms before restart", log_prefix_, backoff.count());
+            retry_backoff_.expires_after(backoff);
+            retry_backoff_.async_wait([self = shared_from_this()](std::error_code ec) mutable {
+                if (ec == asio::error::operation_aborted || self->stopped_) {
+                    return;
+                }
+                self->origin_.restart();
+                self->initiate_bootstrap();
+            });
+            return;
+        }
+        std::string service;
+        std::tie(bootstrap_hostname_, service) = origin_.next_address();
+        log_prefix_ = fmt::format(
+          "[{}/{}/{}/{}] <{}:{}>", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"), bootstrap_hostname_, service);
+        spdlog::debug("{} attempt to establish MCBP connection", log_prefix_);
+        resolver_.async_resolve(bootstrap_hostname_,
+                                service,
+                                std::bind(&mcbp_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
     }
 
     [[nodiscard]] const std::string& id() const
@@ -476,14 +535,26 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             return;
         }
         stopped_ = true;
-        deadline_timer_.cancel();
+        bootstrap_deadline_.cancel();
+        connection_deadline_.cancel();
+        retry_backoff_.cancel();
         resolver_.cancel();
-        if (socket_.is_open()) {
-            socket_.close();
+        if (stream_->is_open()) {
+            stream_->close();
+        }
+        auto ec = std::make_error_code(error::common_errc::request_canceled);
+        if (!bootstrapped_ && bootstrap_handler_) {
+            bootstrap_handler_(ec, {});
+            bootstrap_handler_ = nullptr;
         }
         if (handler_) {
             handler_->stop();
         }
+        for (auto& handler : command_handlers_) {
+            spdlog::debug("{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
+            handler.second(ec, {});
+        }
+        command_handlers_.clear();
     }
 
     void write(const std::vector<uint8_t>& buf)
@@ -491,6 +562,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
+        std::uint32_t opaque{ 0 };
+        std::memcpy(&opaque, buf.data() + 12, sizeof(opaque));
+        spdlog::debug("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
+        SPDLOG_TRACE("{} MCBP send, opaque={}{:a}", log_prefix_, opaque, spdlog::to_hex(data));
+        std::scoped_lock lock(output_buffer_mutex_);
         output_buffer_.push_back(buf);
     }
 
@@ -516,14 +592,15 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                              std::function<void(std::error_code, io::mcbp_message&&)> handler)
     {
         if (stopped_) {
+            spdlog::warn("{} MCBP cancel operation, while trying to write to closed session opaque={}", log_prefix_, opaque);
+            handler(std::make_error_code(error::common_errc::request_canceled), {});
             return;
         }
-        spdlog::trace(
-          "{} MCBP send, opaque={}{:a}", log_prefix_, endpoint_.address().to_string(), endpoint_.port(), opaque, spdlog::to_hex(data));
         command_handlers_.emplace(opaque, std::move(handler));
-        if (bootstrapped_ && socket_.is_open()) {
+        if (bootstrapped_ && stream_->is_open()) {
             write_and_flush(data);
         } else {
+            std::scoped_lock lock(pending_buffer_mutex_);
             pending_buffer_.push_back(data);
         }
     }
@@ -572,6 +649,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return config_->index_for_this_node();
     }
 
+    [[nodiscard]] const std::string& bootstrap_hostname() const
+    {
+        return bootstrap_hostname_;
+    }
+
     [[nodiscard]] uint32_t next_opaque()
     {
         return ++opaque_;
@@ -717,7 +799,15 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void update_configuration(configuration&& config)
     {
+        if (stopped_) {
+            return;
+        }
         if (!config_ || config.rev > config_->rev) {
+            for (auto& node : config.nodes) {
+                if (node.this_node && node.hostname.empty()) {
+                    node.hostname = endpoint_address_;
+                }
+            }
             config_.emplace(config);
             spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
         }
@@ -730,6 +820,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     void update_collection_uid(const std::string& path, std::uint32_t uid)
     {
+        if (stopped_) {
+            return;
+        }
         collection_cache_.update(path, uid);
     }
 
@@ -737,13 +830,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     void invoke_bootstrap_handler(std::error_code ec)
     {
         if (!bootstrapped_ && bootstrap_handler_) {
+            bootstrap_deadline_.cancel();
             bootstrap_handler_(ec, config_.value_or(configuration{}));
             bootstrap_handler_ = nullptr;
         }
-        bootstrapped_ = true;
         if (ec) {
-            stop();
+            return stop();
         }
+        bootstrapped_ = true;
+        handler_ = std::make_unique<normal_handler>(shared_from_this());
+        std::scoped_lock lock(pending_buffer_mutex_);
         if (!pending_buffer_.empty()) {
             for (const auto& buf : pending_buffer_) {
                 write(buf);
@@ -760,11 +856,12 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         if (ec) {
             spdlog::error("{} error on resolve: {}", log_prefix_, ec.message());
-            return invoke_bootstrap_handler(std::make_error_code(error::network_errc::resolve_failure));
+            return initiate_bootstrap();
         }
         endpoints_ = endpoints;
         do_connect(endpoints_.begin());
-        deadline_timer_.async_wait(std::bind(&mcbp_session::check_deadline, this, std::placeholders::_1));
+        connection_deadline_.expires_after(timeout_defaults::connect_timeout);
+        connection_deadline_.async_wait(std::bind(&mcbp_session::check_deadline, shared_from_this(), std::placeholders::_1));
     }
 
     void do_connect(asio::ip::tcp::resolver::results_type::iterator it)
@@ -774,12 +871,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         }
         if (it != endpoints_.end()) {
             spdlog::debug("{} connecting to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
-            deadline_timer_.expires_after(timeout_defaults::connect_timeout);
-            socket_.async_connect(it->endpoint(), std::bind(&mcbp_session::on_connect, this, std::placeholders::_1, it));
+            connection_deadline_.expires_after(timeout_defaults::connect_timeout);
+            stream_->async_connect(it->endpoint(), std::bind(&mcbp_session::on_connect, shared_from_this(), std::placeholders::_1, it));
         } else {
-            spdlog::error("{} no more endpoints left to connect", log_prefix_);
-            invoke_bootstrap_handler(std::make_error_code(error::network_errc::no_endpoints_left));
-            stop();
+            spdlog::error("{} no more endpoints left to connect, will try another address", log_prefix_);
+            return initiate_bootstrap();
         }
     }
 
@@ -788,34 +884,39 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
-        if (!socket_.is_open() || ec) {
+        if (!stream_->is_open() || ec) {
+            spdlog::warn(
+              "{} unable to connect to {}:{}: {}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port(), ec.message());
             do_connect(++it);
         } else {
-            socket_.set_option(asio::ip::tcp::no_delay{ true });
-            socket_.set_option(asio::socket_base::keep_alive{ true });
+            stream_->set_options();
             endpoint_ = it->endpoint();
-            spdlog::debug("{} connected to {}:{}", log_prefix_, endpoint_.address().to_string(), it->endpoint().port());
-            log_prefix_ = fmt::format(
-              "[{}/{}/{}] <{}:{}>", client_id_, id_, bucket_name_.value_or("-"), endpoint_.address().to_string(), endpoint_.port());
+            endpoint_address_ = endpoint_.address().to_string();
+            spdlog::debug("{} connected to {}:{}", log_prefix_, endpoint_address_, it->endpoint().port());
+            log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}/{}:{}>",
+                                      stream_->log_prefix(),
+                                      client_id_,
+                                      id_,
+                                      bucket_name_.value_or("-"),
+                                      bootstrap_hostname_,
+                                      endpoint_address_,
+                                      endpoint_.port());
             handler_ = std::make_unique<bootstrap_handler>(shared_from_this());
-            deadline_timer_.expires_at(asio::steady_timer::time_point::max());
-            deadline_timer_.cancel();
+            connection_deadline_.expires_at(asio::steady_timer::time_point::max());
+            connection_deadline_.cancel();
         }
     }
 
     void check_deadline(std::error_code ec)
     {
-        if (ec == asio::error::operation_aborted) {
-            return;
-        }
-        if (stopped_) {
+        if (ec == asio::error::operation_aborted || stopped_) {
             return;
         }
-        if (deadline_timer_.expiry() <= asio::steady_timer::clock_type::now()) {
-            socket_.close();
-            deadline_timer_.expires_at(asio::steady_timer::time_point::max());
+        if (connection_deadline_.expiry() <= asio::steady_timer::clock_type::now()) {
+            stream_->close();
+            connection_deadline_.expires_at(asio::steady_timer::time_point::max());
         }
-        deadline_timer_.async_wait(std::bind(&mcbp_session::check_deadline, this, std::placeholders::_1));
+        connection_deadline_.async_wait(std::bind(&mcbp_session::check_deadline, shared_from_this(), std::placeholders::_1));
     }
 
     void do_read()
@@ -827,36 +928,44 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             return;
         }
         reading_ = true;
-        socket_.async_read_some(asio::buffer(input_buffer_),
-                                [self = shared_from_this()](std::error_code ec, std::size_t bytes_transferred) {
-                                    if (ec == asio::error::operation_aborted || self->stopped_) {
-                                        return;
-                                    }
-                                    if (ec) {
-                                        spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
-                                        return self->stop();
-                                    }
-                                    self->parser_.feed(self->input_buffer_.data(), self->input_buffer_.data() + ssize_t(bytes_transferred));
-
-                                    for (;;) {
-                                        mcbp_message msg{};
-                                        switch (self->parser_.next(msg)) {
-                                            case mcbp_parser::ok:
-                                                spdlog::trace("{} MCBP recv, opaque={}{:a}{:a}",
-                                                              self->log_prefix_,
-                                                              msg.header.opaque,
-                                                              spdlog::to_hex(msg.header_data()),
-                                                              spdlog::to_hex(msg.body));
-                                                self->handler_->handle(std::move(msg));
-                                                break;
-                                            case mcbp_parser::need_data:
-                                                self->reading_ = false;
-                                                return self->do_read();
-                                            case mcbp_parser::failure:
-                                                return self->stop();
-                                        }
-                                    }
-                                });
+        stream_->async_read_some(
+          asio::buffer(input_buffer_), [self = shared_from_this()](std::error_code ec, std::size_t bytes_transferred) {
+              if (ec == asio::error::operation_aborted || self->stopped_) {
+                  return;
+              }
+              if (ec) {
+                  spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
+                  return self->stop();
+              }
+              self->parser_.feed(self->input_buffer_.data(), self->input_buffer_.data() + ssize_t(bytes_transferred));
+
+              for (;;) {
+                  mcbp_message msg{};
+                  switch (self->parser_.next(msg)) {
+                      case mcbp_parser::ok:
+                          spdlog::debug(
+                            "{} MCBP recv, opaque={}, {:n}", self->log_prefix_, msg.header.opaque, spdlog::to_hex(msg.header_data()));
+                          SPDLOG_TRACE("{} MCBP recv, opaque={}{:a}{:a}",
+                                       self->log_prefix_,
+                                       msg.header.opaque,
+                                       spdlog::to_hex(msg.header_data()),
+                                       spdlog::to_hex(msg.body));
+                          self->handler_->handle(std::move(msg));
+                          if (self->stopped_) {
+                              return;
+                          }
+                          break;
+                      case mcbp_parser::need_data:
+                          self->reading_ = false;
+                          if (!self->stopped_) {
+                              self->do_read();
+                          }
+                          return;
+                      case mcbp_parser::failure:
+                          return self->stop();
+                  }
+              }
+          });
     }
 
     void do_write()
@@ -864,7 +973,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return;
         }
-        if (!writing_buffer_.empty()) {
+        std::scoped_lock lock(writing_buffer_mutex_, output_buffer_mutex_);
+        if (!writing_buffer_.empty() || output_buffer_.empty()) {
             return;
         }
         std::swap(writing_buffer_, output_buffer_);
@@ -873,18 +983,19 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         for (auto& buf : writing_buffer_) {
             buffers.emplace_back(asio::buffer(buf));
         }
-        asio::async_write(socket_, buffers, [self = shared_from_this()](std::error_code ec, std::size_t /*unused*/) {
-            if (self->stopped_) {
+        stream_->async_write(buffers, [self = shared_from_this()](std::error_code ec, std::size_t /*unused*/) {
+            if (ec == asio::error::operation_aborted || self->stopped_) {
                 return;
             }
             if (ec) {
                 spdlog::error("{} IO error while writing to the socket: {}", self->log_prefix_, ec.message());
                 return self->stop();
             }
-            self->writing_buffer_.clear();
-            if (!self->output_buffer_.empty()) {
-                self->do_write();
+            {
+                std::scoped_lock inner_lock(self->writing_buffer_mutex_);
+                self->writing_buffer_.clear();
             }
+            self->do_write();
             self->do_read();
         });
     }
@@ -893,13 +1004,15 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     std::string id_;
     asio::io_context& ctx_;
     asio::ip::tcp::resolver resolver_;
-    asio::strand<asio::io_context::executor_type> strand_;
-    asio::ip::tcp::socket socket_;
-    asio::steady_timer deadline_timer_;
+    std::unique_ptr<stream_impl> stream_;
+    asio::steady_timer bootstrap_deadline_;
+    asio::steady_timer connection_deadline_;
+    asio::steady_timer retry_backoff_;
+    couchbase::origin origin_;
     std::optional<std::string> bucket_name_;
     mcbp_parser parser_;
     std::unique_ptr<message_handler> handler_;
-    std::function<void(std::error_code, configuration)> bootstrap_handler_;
+    std::function<void(std::error_code, const configuration&)> bootstrap_handler_{};
     std::map<uint32_t, std::function<void(std::error_code, io::mcbp_message&&)>> command_handlers_{};
 
     bool bootstrapped_{ false };
@@ -910,14 +1023,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 
     std::atomic<std::uint32_t> opaque_{ 0 };
 
-    std::string username_;
-    std::string password_;
-
     std::array<std::uint8_t, 16384> input_buffer_{};
     std::vector<std::vector<std::uint8_t>> output_buffer_{};
     std::vector<std::vector<std::uint8_t>> pending_buffer_{};
     std::vector<std::vector<std::uint8_t>> writing_buffer_{};
+    std::mutex output_buffer_mutex_{};
+    std::mutex pending_buffer_mutex_{};
+    std::mutex writing_buffer_mutex_{};
+    std::string bootstrap_hostname_{};
     asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
+    std::string endpoint_address_{};     // cached string with endpoint address
     asio::ip::tcp::resolver::results_type endpoints_;
     std::vector<protocol::hello_feature> supported_features_;
     std::optional<configuration> config_;