couchbase 3.0.0.alpha.5 → 3.0.0.beta.1

data/ext/couchbase/io/dns_client.hxx

@@ -119,7 +119,9 @@ class dns_client
                     return;
                 }
                 self->udp_.cancel();
-                self->tcp_.cancel();
+                if (self->tcp_.is_open()) {
+                    self->tcp_.cancel();
+                }
             });
         }
 

data/ext/couchbase/io/http_command.hxx

@@ -0,0 +1,91 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <io/http_session.hxx>
+
+namespace couchbase::operations
+{
+
+template<typename Request>
+struct http_command : public std::enable_shared_from_this<http_command<Request>> {
+    using encoded_request_type = typename Request::encoded_request_type;
+    using encoded_response_type = typename Request::encoded_response_type;
+    asio::steady_timer deadline;
+    asio::steady_timer retry_backoff;
+    Request request;
+    encoded_request_type encoded;
+
+    http_command(asio::io_context& ctx, Request req)
+      : deadline(ctx)
+      , retry_backoff(ctx)
+      , request(req)
+    {
+    }
+
+    template<typename Handler>
+    void send_to(std::shared_ptr<io::http_session> session, Handler&& handler)
+    {
+        encoded.type = Request::type;
+        request.encode_to(encoded);
+        encoded.headers["client-context-id"] = request.client_context_id;
+        auto log_prefix = session->log_prefix();
+        spdlog::debug("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms",
+                      log_prefix,
+                      encoded.type,
+                      encoded.method,
+                      encoded.path,
+                      request.client_context_id,
+                      request.timeout.count());
+        SPDLOG_TRACE("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms{:a}",
+                     log_prefix,
+                     encoded.type,
+                     encoded.method,
+                     encoded.path,
+                     request.client_context_id,
+                     request.timeout.count(),
+                     spdlog::to_hex(encoded.body));
+        session->write_and_subscribe(encoded,
+                                     [self = this->shared_from_this(), log_prefix, handler = std::forward<Handler>(handler)](
+                                       std::error_code ec, io::http_response&& msg) mutable {
+                                         self->deadline.cancel();
+                                         encoded_response_type resp(msg);
+                                         spdlog::debug("{} HTTP response: {}, client_context_id={}, status={}",
+                                                       log_prefix,
+                                                       self->request.type,
+                                                       self->request.client_context_id,
+                                                       resp.status_code);
+                                         SPDLOG_TRACE("{} HTTP response: {}, client_context_id={}, status={}{:a}",
+                                                      log_prefix,
+                                                      self->request.type,
+                                                      self->request.client_context_id,
+                                                      resp.status_code,
+                                                      spdlog::to_hex(resp.body));
+                                         handler(make_response(ec, self->request, resp));
+                                     });
+        deadline.expires_after(request.timeout);
+        deadline.async_wait([session](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            session->stop();
+        });
+    }
+};
+
+} // namespace couchbase::operations

data/ext/couchbase/io/http_session.hxx

@@ -51,8 +51,7 @@ class http_session : public std::enable_shared_from_this<http_session>
       , id_(uuid::to_string(uuid::random()))
       , ctx_(ctx)
       , resolver_(ctx_)
-      , strand_(asio::make_strand(ctx_))
-      , socket_(strand_)
+      , stream_(std::make_unique<plain_stream_impl>(ctx_))
       , deadline_timer_(ctx_)
       , username_(username)
       , password_(password)
@@ -66,10 +65,37 @@ class http_session : public std::enable_shared_from_this<http_session>
                                 client_id_,
                                 id_,
                                 BACKEND_SYSTEM))
+      , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
+    {
+    }
+
+    http_session(const std::string& client_id,
+                 asio::io_context& ctx,
+                 asio::ssl::context& tls,
+                 const std::string& username,
+                 const std::string& password,
+                 const std::string& hostname,
+                 const std::string& service)
+      : client_id_(client_id)
+      , id_(uuid::to_string(uuid::random()))
+      , ctx_(ctx)
+      , resolver_(ctx_)
+      , stream_(std::make_unique<tls_stream_impl>(ctx_, tls))
+      , deadline_timer_(ctx_)
+      , username_(username)
+      , password_(password)
+      , hostname_(hostname)
+      , service_(service)
+      , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
+                                BACKEND_VERSION_MAJOR,
+                                BACKEND_VERSION_MINOR,
+                                BACKEND_VERSION_PATCH,
+                                BACKEND_GIT_REVISION,
+                                client_id_,
+                                id_,
+                                BACKEND_SYSTEM))
+      , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
     {
-        log_prefix_ = fmt::format("[{}/{}]", client_id_, id_);
-        resolver_.async_resolve(
-          hostname, service, std::bind(&http_session::on_resolve, this, std::placeholders::_1, std::placeholders::_2));
     }
 
     ~http_session()
@@ -77,6 +103,12 @@ class http_session : public std::enable_shared_from_this<http_session>
         stop();
     }
 
+    void start()
+    {
+        resolver_.async_resolve(
+          hostname_, service_, std::bind(&http_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
+    }
+
     [[nodiscard]] const std::string& log_prefix() const
     {
         return log_prefix_;
@@ -100,8 +132,8 @@ class http_session : public std::enable_shared_from_this<http_session>
     void stop()
     {
         stopped_ = true;
-        if (socket_.is_open()) {
-            socket_.close();
+        if (stream_->is_open()) {
+            stream_->close();
         }
         deadline_timer_.cancel();
 
@@ -116,6 +148,11 @@ class http_session : public std::enable_shared_from_this<http_session>
         }
     }
 
+    bool keep_alive()
+    {
+        return keep_alive_;
+    }
+
     bool is_stopped()
     {
         return stopped_;
@@ -177,7 +214,7 @@ class http_session : public std::enable_shared_from_this<http_session>
         }
         endpoints_ = endpoints;
         do_connect(endpoints_.begin());
-        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, this, std::placeholders::_1));
+        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1));
     }
 
     void do_connect(asio::ip::tcp::resolver::results_type::iterator it)
@@ -185,7 +222,7 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (it != endpoints_.end()) {
             spdlog::debug("{} connecting to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
             deadline_timer_.expires_after(timeout_defaults::connect_timeout);
-            socket_.async_connect(it->endpoint(), std::bind(&http_session::on_connect, this, std::placeholders::_1, it));
+            stream_->async_connect(it->endpoint(), std::bind(&http_session::on_connect, shared_from_this(), std::placeholders::_1, it));
         } else {
             spdlog::error("{} no more endpoints left to connect", log_prefix_);
             stop();
@@ -197,7 +234,9 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (stopped_) {
             return;
         }
-        if (!socket_.is_open() || ec) {
+        if (!stream_->is_open() || ec) {
+            spdlog::warn(
+              "{} unable to connect to {}:{}: {}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port(), ec.message());
             do_connect(++it);
         } else {
             connected_ = true;
@@ -219,10 +258,10 @@ class http_session : public std::enable_shared_from_this<http_session>
             return;
         }
         if (deadline_timer_.expiry() <= asio::steady_timer::clock_type::now()) {
-            socket_.close();
+            stream_->close();
             deadline_timer_.expires_at(asio::steady_timer::time_point::max());
         }
-        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, this, std::placeholders::_1));
+        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1));
     }
 
     void do_read()
@@ -230,12 +269,12 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (stopped_) {
             return;
         }
-        socket_.async_read_some(
+        stream_->async_read_some(
           asio::buffer(input_buffer_), [self = shared_from_this()](std::error_code ec, std::size_t bytes_transferred) {
-              if (self->stopped_) {
+              if (ec == asio::error::operation_aborted || self->stopped_) {
                   return;
               }
-              if (ec && ec != asio::error::operation_aborted) {
+              if (ec) {
                   spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
                   return self->stop();
               }
@@ -273,8 +312,8 @@ class http_session : public std::enable_shared_from_this<http_session>
         for (auto& buf : writing_buffer_) {
             buffers.emplace_back(asio::buffer(buf));
         }
-        asio::async_write(socket_, buffers, [self = shared_from_this()](std::error_code ec, std::size_t /* bytes_transferred */) {
-            if (self->stopped_) {
+        stream_->async_write(buffers, [self = shared_from_this()](std::error_code ec, std::size_t /* bytes_transferred */) {
+            if (ec == asio::error::operation_aborted || self->stopped_) {
                 return;
             }
             if (ec) {
@@ -293,8 +332,7 @@ class http_session : public std::enable_shared_from_this<http_session>
     std::string id_;
     asio::io_context& ctx_;
     asio::ip::tcp::resolver resolver_;
-    asio::strand<asio::io_context::executor_type> strand_;
-    asio::ip::tcp::socket socket_;
+    std::unique_ptr<stream_impl> stream_;
     asio::steady_timer deadline_timer_;
 
     std::string username_;
@@ -305,6 +343,7 @@ class http_session : public std::enable_shared_from_this<http_session>
 
     bool stopped_{ false };
     bool connected_{ false };
+    bool keep_alive_{ false };
 
     std::function<void()> on_stop_handler_{ nullptr };
 

data/ext/couchbase/io/http_session_manager.hxx

@@ -28,14 +28,16 @@ namespace couchbase::io
 class http_session_manager : public std::enable_shared_from_this<http_session_manager>
 {
   public:
-    http_session_manager(const std::string& client_id, asio::io_context& ctx)
+    http_session_manager(const std::string& client_id, asio::io_context& ctx, asio::ssl::context& tls)
       : client_id_(client_id)
       , ctx_(ctx)
+      , tls_(tls)
     {
     }
 
-    void set_configuration(const configuration& config)
+    void set_configuration(const configuration& config, const cluster_options& options)
     {
+        options_ = options;
         config_ = config;
         next_index_ = 0;
         if (config_.nodes.size() > 1) {
@@ -48,6 +50,7 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
 
     std::shared_ptr<http_session> check_out(service_type type, const std::string& username, const std::string& password)
     {
+        std::scoped_lock lock(sessions_mutex_);
         if (idle_sessions_[type].empty()) {
             std::string hostname;
             std::uint16_t port = 0;
@@ -56,8 +59,15 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
                 return nullptr;
             }
             config_.nodes.size();
-            auto session = std::make_shared<http_session>(client_id_, ctx_, username, password, hostname, std::to_string(port));
+            std::shared_ptr<http_session> session;
+            if (options_.enable_tls) {
+                session = std::make_shared<http_session>(client_id_, ctx_, tls_, username, password, hostname, std::to_string(port));
+            } else {
+                session = std::make_shared<http_session>(client_id_, ctx_, username, password, hostname, std::to_string(port));
+            }
+            session->start();
             session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {
+                std::scoped_lock inner_lock(self->sessions_mutex_);
                 self->busy_sessions_[type].remove_if([id](const auto& s) -> bool { return s->id() == id; });
                 self->idle_sessions_[type].remove_if([id](const auto& s) -> bool { return s->id() == id; });
             });
@@ -72,7 +82,14 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
 
     void check_in(service_type type, std::shared_ptr<http_session> session)
     {
-        idle_sessions_[type].push_back(session);
+        if (!session->keep_alive()) {
+            return session->stop();
+        }
+        if (!session->is_stopped()) {
+            std::scoped_lock lock(sessions_mutex_);
+            spdlog::debug("{} put HTTP session back to idle connections", session->log_prefix());
+            idle_sessions_[type].push_back(session);
+        }
     }
 
   private:
@@ -83,34 +100,9 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
             --candidates;
             auto& node = config_.nodes[next_index_];
             next_index_ = (next_index_ + 1) % config_.nodes.size();
-            std::uint16_t port = 0;
-            switch (type) {
-                case service_type::query:
-                    port = node.services_plain.query.value_or(0);
-                    break;
-
-                case service_type::analytics:
-                    port = node.services_plain.analytics.value_or(0);
-                    break;
-
-                case service_type::search:
-                    port = node.services_plain.search.value_or(0);
-                    break;
-
-                case service_type::views:
-                    port = node.services_plain.views.value_or(0);
-                    break;
-
-                case service_type::management:
-                    port = node.services_plain.management.value_or(0);
-                    break;
-
-                case service_type::kv:
-                    port = node.services_plain.key_value.value_or(0);
-                    break;
-            }
+            std::uint16_t port = node.port_or(options_.network, type, options_.enable_tls, 0);
             if (port != 0) {
-                return { node.hostname, port };
+                return { node.hostname_for(options_.network), port };
             }
         }
         return { "", 0 };
@@ -118,10 +110,13 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
 
     std::string client_id_;
     asio::io_context& ctx_;
+    asio::ssl::context& tls_;
+    cluster_options options_;
 
     configuration config_{};
     std::map<service_type, std::list<std::shared_ptr<http_session>>> busy_sessions_{};
     std::map<service_type, std::list<std::shared_ptr<http_session>>> idle_sessions_{};
     std::size_t next_index_{ 0 };
+    std::mutex sessions_mutex_{};
 };
 } // namespace couchbase::io

data/ext/couchbase/io/mcbp_command.hxx

@@ -0,0 +1,180 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <io/mcbp_session.hxx>
+#include <protocol/cmd_get_collection_id.hxx>
+#include <functional>
+#include <utility>
+
+namespace couchbase::operations
+{
+
+using mcbp_command_handler = std::function<void(std::error_code, std::optional<io::mcbp_message>)>;
+
+template<typename Request>
+struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>> {
+    using encoded_request_type = typename Request::encoded_request_type;
+    using encoded_response_type = typename Request::encoded_response_type;
+    asio::steady_timer deadline;
+    asio::steady_timer retry_backoff;
+    Request request;
+    encoded_request_type encoded;
+    std::optional<std::uint32_t> opaque_{};
+    std::shared_ptr<io::mcbp_session> session_{};
+    mcbp_command_handler handler_{};
+
+    mcbp_command(asio::io_context& ctx, Request req)
+      : deadline(ctx)
+      , retry_backoff(ctx)
+      , request(req)
+    {
+    }
+
+    void start(mcbp_command_handler&& handler)
+    {
+        handler_ = handler;
+        deadline.expires_after(request.timeout);
+        deadline.async_wait([self = this->shared_from_this()](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            self->cancel();
+        });
+    }
+
+    void cancel()
+    {
+        if (opaque_ && session_) {
+            session_->cancel(opaque_.value(), asio::error::operation_aborted);
+        }
+        handler_ = nullptr;
+    }
+
+    void invoke_handler(std::error_code ec, std::optional<io::mcbp_message> msg = {})
+    {
+        if (handler_) {
+            handler_(ec, std::move(msg));
+        }
+        handler_ = nullptr;
+    }
+
+    void request_collection_id()
+    {
+        protocol::client_request<protocol::get_collection_id_request_body> req;
+        req.opaque(session_->next_opaque());
+        req.body().collection_path(request.id.collection);
+        session_->write_and_subscribe(req.opaque(),
+                                      req.data(session_->supports_feature(protocol::hello_feature::snappy)),
+                                      [self = this->shared_from_this()](std::error_code ec, io::mcbp_message&& msg) mutable {
+                                          if (ec == asio::error::operation_aborted) {
+                                              return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
+                                          }
+                                          if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
+                                              if (self->request.id.collection_uid) {
+                                                  return self->handle_unknown_collection();
+                                              }
+                                              return self->invoke_handler(ec);
+                                          }
+                                          if (ec) {
+                                              return self->invoke_handler(ec);
+                                          }
+                                          protocol::client_response<protocol::get_collection_id_response_body> resp(msg);
+                                          self->session_->update_collection_uid(self->request.id.collection, resp.body().collection_uid());
+                                          self->request.id.collection_uid = resp.body().collection_uid();
+                                          return self->send();
+                                      });
+    }
+
+    void handle_unknown_collection()
+    {
+        auto backoff = std::chrono::milliseconds(500);
+        auto time_left = deadline.expiry() - std::chrono::steady_clock::now();
+        spdlog::debug("{} unknown collection response for \"{}/{}/{}\", time_left={}ms",
+                      session_->log_prefix(),
+                      request.id.bucket,
+                      request.id.collection,
+                      request.id.key,
+                      std::chrono::duration_cast<std::chrono::milliseconds>(time_left).count());
+        if (time_left < backoff) {
+            return invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
+        }
+        retry_backoff.expires_after(backoff);
+        retry_backoff.async_wait([self = this->shared_from_this()](std::error_code ec) mutable {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            self->request_collection_id();
+        });
+    }
+
+    void send()
+    {
+        opaque_ = session_->next_opaque();
+        request.opaque = *opaque_;
+        if (!request.id.collection_uid) {
+            if (session_->supports_feature(protocol::hello_feature::collections)) {
+                auto collection_id = session_->get_collection_uid(request.id.collection);
+                if (collection_id) {
+                    request.id.collection_uid = *collection_id;
+                } else {
+                    spdlog::debug("{} no cache entry for collection, resolve collection id for \"{}/{}/{}\", timeout={}ms",
+                                  session_->log_prefix(),
+                                  request.id.bucket,
+                                  request.id.collection,
+                                  request.id.key,
+                                  request.timeout.count());
+                    return request_collection_id();
+                }
+            } else {
+                if (!request.id.collection.empty() && request.id.collection != "_default._default") {
+                    return invoke_handler(std::make_error_code(error::common_errc::unsupported_operation));
+                }
+            }
+        }
+        request.encode_to(encoded);
+
+        session_->write_and_subscribe(request.opaque,
+                                      encoded.data(session_->supports_feature(protocol::hello_feature::snappy)),
+                                      [self = this->shared_from_this()](std::error_code ec, io::mcbp_message&& msg) mutable {
+                                          self->retry_backoff.cancel();
+                                          if (ec == asio::error::operation_aborted) {
+                                              return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
+                                          }
+                                          if (ec == std::make_error_code(error::common_errc::request_canceled)) {
+                                              return self->invoke_handler(ec);
+                                          }
+                                          if (msg.header.status() == static_cast<std::uint16_t>(protocol::status::unknown_collection)) {
+                                              return self->handle_unknown_collection();
+                                          }
+                                          self->deadline.cancel();
+                                          self->invoke_handler(ec, msg);
+                                      });
+    }
+
+    void send_to(std::shared_ptr<io::mcbp_session> session)
+    {
+        if (!handler_) {
+            return;
+        }
+        session_ = std::move(session);
+        send();
+    }
+};
+
+} // namespace couchbase::operations