couchbase 3.0.0.alpha.5 → 3.0.3

data/ext/couchbase/io/dns_codec.hxx

@@ -19,8 +19,7 @@
 
 #include <cstring>
 
-#include <arpa/inet.h>
-
+#include <asio/ip/udp.hpp>
 #include <io/dns_message.hxx>
 
 namespace couchbase::io::dns

data/ext/couchbase/io/http_command.hxx

@@ -0,0 +1,111 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <io/http_session.hxx>
+
+namespace couchbase::operations
+{
+
+template<typename Request>
+struct http_command : public std::enable_shared_from_this<http_command<Request>> {
+    using encoded_request_type = typename Request::encoded_request_type;
+    using encoded_response_type = typename Request::encoded_response_type;
+    using error_context_type = typename Request::error_context_type;
+    asio::steady_timer deadline;
+    asio::steady_timer retry_backoff;
+    Request request;
+    encoded_request_type encoded;
+
+    http_command(asio::io_context& ctx, Request req)
+      : deadline(ctx)
+      , retry_backoff(ctx)
+      , request(req)
+    {
+    }
+
+    template<typename Handler>
+    void send_to(std::shared_ptr<io::http_session> session, Handler&& handler)
+    {
+        encoded.type = request.type;
+        auto encoding_ec = request.encode_to(encoded, session->http_context());
+        if (encoding_ec) {
+            error_context_type ctx{};
+            ctx.ec = encoding_ec;
+            ctx.client_context_id = request.client_context_id;
+            return handler(make_response(std::move(ctx), request, {}));
+        }
+        encoded.headers["client-context-id"] = request.client_context_id;
+        auto log_prefix = session->log_prefix();
+        spdlog::trace(R"({} HTTP request: {}, method={}, path="{}", client_context_id="{}", timeout={}ms)",
+                      log_prefix,
+                      encoded.type,
+                      encoded.method,
+                      encoded.path,
+                      request.client_context_id,
+                      request.timeout.count());
+        SPDLOG_TRACE(R"({} HTTP request: {}, method={}, path="{}", client_context_id="{}", timeout={}ms{:a})",
+                     log_prefix,
+                     encoded.type,
+                     encoded.method,
+                     encoded.path,
+                     request.client_context_id,
+                     request.timeout.count(),
+                     spdlog::to_hex(encoded.body));
+        session->write_and_subscribe(encoded,
+                                     [self = this->shared_from_this(), log_prefix, session, handler = std::forward<Handler>(handler)](
+                                       std::error_code ec, io::http_response&& msg) mutable {
+                                         self->deadline.cancel();
+                                         encoded_response_type resp(msg);
+                                         spdlog::trace(R"({} HTTP response: {}, client_context_id="{}", status={})",
+                                                       log_prefix,
+                                                       self->request.type,
+                                                       self->request.client_context_id,
+                                                       resp.status_code);
+                                         SPDLOG_TRACE(R"({} HTTP response: {}, client_context_id="{}", status={}{:a})",
+                                                      log_prefix,
+                                                      self->request.type,
+                                                      self->request.client_context_id,
+                                                      resp.status_code,
+                                                      spdlog::to_hex(resp.body));
+                                         try {
+                                             error_context_type ctx{};
+                                             ctx.ec = ec;
+                                             ctx.client_context_id = self->request.client_context_id;
+                                             ctx.method = self->encoded.method;
+                                             ctx.path = self->encoded.path;
+                                             ctx.last_dispatched_from = session->local_address();
+                                             ctx.last_dispatched_to = session->remote_address();
+                                             ctx.http_status = msg.status_code;
+                                             ctx.http_body = msg.body;
+                                             handler(make_response(std::move(ctx), self->request, std::move(msg)));
+                                         } catch (priv::retry_http_request&) {
+                                             self->send_to(session, std::forward<Handler>(handler));
+                                         }
+                                     });
+        deadline.expires_after(request.timeout);
+        deadline.async_wait([session](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            session->stop();
+        });
+    }
+};
+
+} // namespace couchbase::operations

data/ext/couchbase/io/http_context.hxx

@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <configuration.hxx>
+#include <io/query_cache.hxx>
+
+namespace couchbase
+{
+struct http_context {
+    const configuration& config;
+    const cluster_options& options;
+    query_cache& cache;
+};
+
+namespace priv
+{
+class retry_http_request : public std::exception
+{
+};
+} // namespace priv
+} // namespace couchbase

data/ext/couchbase/io/http_message.hxx

@@ -34,5 +34,14 @@ struct http_response {
     std::string status_message;
     std::map<std::string, std::string> headers;
     std::string body;
+
+    [[nodiscard]] bool must_close_connection() const
+    {
+        const auto it = headers.find("connection");
+        if (it != headers.end()) {
+            return it->second == "close";
+        }
+        return false;
+    }
 };
 } // namespace couchbase::io

data/ext/couchbase/io/http_parser.hxx

@@ -19,6 +19,7 @@
 
 #include <http_parser.h>
 #include <io/http_message.hxx>
+#include <algorithm>
 
 namespace couchbase::io
 {
@@ -106,6 +107,7 @@ struct http_parser {
     int on_header_field(const char* at, std::size_t length)
     {
         header_field.assign(at, length);
+        std::transform(header_field.begin(), header_field.end(), header_field.begin(), [](unsigned char c) { return std::tolower(c); });
         return 0;
     }
 

data/ext/couchbase/io/http_session.hxx

@@ -17,8 +17,8 @@
 
 #pragma once
 
-#include <utility>
 #include <memory>
+#include <utility>
 
 #include <spdlog/spdlog.h>
 
@@ -30,8 +30,9 @@
 #include <errors.hxx>
 #include <version.hxx>
 
-#include <io/http_parser.hxx>
+#include <io/http_context.hxx>
 #include <io/http_message.hxx>
+#include <io/http_parser.hxx>
 #include <platform/base64.h>
 #include <timeout_defaults.hxx>
 
@@ -41,35 +42,53 @@ namespace couchbase::io
 class http_session : public std::enable_shared_from_this<http_session>
 {
   public:
-    http_session(const std::string& client_id,
+    http_session(service_type type,
+                 const std::string& client_id,
                  asio::io_context& ctx,
-                 const std::string& username,
-                 const std::string& password,
+                 const cluster_credentials& credentials,
                  const std::string& hostname,
-                 const std::string& service)
-      : client_id_(client_id)
+                 const std::string& service,
+                 http_context http_ctx)
+      : type_(type)
+      , client_id_(client_id)
       , id_(uuid::to_string(uuid::random()))
       , ctx_(ctx)
       , resolver_(ctx_)
-      , strand_(asio::make_strand(ctx_))
-      , socket_(strand_)
+      , stream_(std::make_unique<plain_stream_impl>(ctx_))
       , deadline_timer_(ctx_)
-      , username_(username)
-      , password_(password)
+      , idle_timer_(ctx_)
+      , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
-      , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
-                                BACKEND_VERSION_MAJOR,
-                                BACKEND_VERSION_MINOR,
-                                BACKEND_VERSION_PATCH,
-                                BACKEND_GIT_REVISION,
-                                client_id_,
-                                id_,
-                                BACKEND_SYSTEM))
+      , user_agent_(fmt::format("{}; client/{}; session/{}; {}", couchbase::sdk_id(), client_id_, id_, BACKEND_SYSTEM))
+      , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
+      , http_ctx_(std::move(http_ctx))
+    {
+    }
+
+    http_session(service_type type,
+                 const std::string& client_id,
+                 asio::io_context& ctx,
+                 asio::ssl::context& tls,
+                 const cluster_credentials& credentials,
+                 const std::string& hostname,
+                 const std::string& service,
+                 http_context http_ctx)
+      : type_(type)
+      , client_id_(client_id)
+      , id_(uuid::to_string(uuid::random()))
+      , ctx_(ctx)
+      , resolver_(ctx_)
+      , stream_(std::make_unique<tls_stream_impl>(ctx_, tls))
+      , deadline_timer_(ctx_)
+      , idle_timer_(ctx_)
+      , credentials_(credentials)
+      , hostname_(hostname)
+      , service_(service)
+      , user_agent_(fmt::format("{}; client/{}; session/{}; {}", couchbase::sdk_id(), client_id_, id_, BACKEND_SYSTEM))
+      , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
+      , http_ctx_(std::move(http_ctx))
     {
-        log_prefix_ = fmt::format("[{}/{}]", client_id_, id_);
-        resolver_.async_resolve(
-          hostname, service, std::bind(&http_session::on_resolve, this, std::placeholders::_1, std::placeholders::_2));
     }
 
     ~http_session()
@@ -77,6 +96,46 @@ class http_session : public std::enable_shared_from_this<http_session>
         stop();
     }
 
+    [[nodiscard]] couchbase::http_context& http_context()
+    {
+        return http_ctx_;
+    }
+
+    std::string remote_address() const
+    {
+        if (endpoint_.protocol() == asio::ip::tcp::v6()) {
+            return fmt::format("[{}]:{}", endpoint_address_, endpoint_.port());
+        }
+        return fmt::format("{}:{}", endpoint_address_, endpoint_.port());
+    }
+
+    std::string local_address() const
+    {
+        if (endpoint_.protocol() == asio::ip::tcp::v6()) {
+            return fmt::format("[{}]:{}", local_endpoint_address_, local_endpoint_.port());
+        }
+        return fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port());
+    }
+
+    [[nodiscard]] diag::endpoint_diag_info diag_info() const
+    {
+        return { type_,
+                 id_,
+                 last_active_.time_since_epoch().count() == 0 ? std::nullopt
+                                                              : std::make_optional(std::chrono::duration_cast<std::chrono::microseconds>(
+                                                                  std::chrono::steady_clock::now() - last_active_)),
+                 remote_address(),
+                 local_address(),
+                 state_ };
+    }
+
+    void start()
+    {
+        state_ = diag::endpoint_state::connecting;
+        resolver_.async_resolve(
+          hostname_, service_, std::bind(&http_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
+    }
+
     [[nodiscard]] const std::string& log_prefix() const
     {
         return log_prefix_;
@@ -99,24 +158,38 @@ class http_session : public std::enable_shared_from_this<http_session>
 
     void stop()
     {
+        if (stopped_) {
+            return;
+        }
         stopped_ = true;
-        if (socket_.is_open()) {
-            socket_.close();
+        state_ = diag::endpoint_state::disconnecting;
+        if (stream_->is_open()) {
+            stream_->close();
         }
         deadline_timer_.cancel();
+        idle_timer_.cancel();
 
-        for (auto& handler : command_handlers_) {
-            handler(std::make_error_code(error::common_errc::ambiguous_timeout), {});
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            for (auto& handler : command_handlers_) {
+                handler(std::make_error_code(error::common_errc::ambiguous_timeout), {});
+            }
+            command_handlers_.clear();
         }
-        command_handlers_.clear();
 
         if (on_stop_handler_) {
             on_stop_handler_();
             on_stop_handler_ = nullptr;
         }
+        state_ = diag::endpoint_state::disconnected;
+    }
+
+    bool keep_alive() const
+    {
+        return keep_alive_;
     }
 
-    bool is_stopped()
+    bool is_stopped() const
     {
         return stopped_;
     }
@@ -148,13 +221,17 @@ class http_session : public std::enable_shared_from_this<http_session>
         do_write();
     }
 
-    void write_and_subscribe(io::http_request& request, std::function<void(std::error_code, io::http_response&&)> handler)
+    void write_and_subscribe(io::http_request& request, std::function<void(std::error_code, io::http_response&&)>&& handler)
     {
         if (stopped_) {
             return;
         }
+        if (request.headers["connection"] == "keep-alive") {
+            keep_alive_ = true;
+        }
         request.headers["user-agent"] = user_agent_;
-        request.headers["authorization"] = fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", username_, password_)));
+        request.headers["authorization"] =
+          fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", credentials_.username, credentials_.password)));
         write(fmt::format("{} {} HTTP/1.1\r\nhost: {}:{}\r\n", request.method, request.path, hostname_, service_));
         if (!request.body.empty()) {
             request.headers["content-length"] = std::to_string(request.body.size());
@@ -164,10 +241,29 @@ class http_session : public std::enable_shared_from_this<http_session>
         }
         write("\r\n");
         write(request.body);
-        command_handlers_.push_back(std::move(handler));
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            command_handlers_.push_back(std::move(handler));
+        }
         flush();
     }
 
+    void set_idle(std::chrono::milliseconds timeout)
+    {
+        idle_timer_.expires_after(timeout);
+        return idle_timer_.async_wait([self = shared_from_this()](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            self->stop();
+        });
+    }
+
+    void reset_idle()
+    {
+        idle_timer_.cancel();
+    }
+
   private:
     void on_resolve(std::error_code ec, const asio::ip::tcp::resolver::results_type& endpoints)
     {
@@ -175,9 +271,10 @@ class http_session : public std::enable_shared_from_this<http_session>
             spdlog::error("{} error on resolve: {}", log_prefix_, ec.message());
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         endpoints_ = endpoints;
         do_connect(endpoints_.begin());
-        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, this, std::placeholders::_1));
+        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1));
     }
 
     void do_connect(asio::ip::tcp::resolver::results_type::iterator it)
@@ -185,7 +282,7 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (it != endpoints_.end()) {
             spdlog::debug("{} connecting to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
             deadline_timer_.expires_after(timeout_defaults::connect_timeout);
-            socket_.async_connect(it->endpoint(), std::bind(&http_session::on_connect, this, std::placeholders::_1, it));
+            stream_->async_connect(it->endpoint(), std::bind(&http_session::on_connect, shared_from_this(), std::placeholders::_1, it));
         } else {
             spdlog::error("{} no more endpoints left to connect", log_prefix_);
             stop();
@@ -197,11 +294,18 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (stopped_) {
             return;
         }
-        if (!socket_.is_open() || ec) {
+        last_active_ = std::chrono::steady_clock::now();
+        if (!stream_->is_open() || ec) {
+            spdlog::warn(
+              "{} unable to connect to {}:{}: {}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port(), ec.message());
             do_connect(++it);
         } else {
+            state_ = diag::endpoint_state::connected;
             connected_ = true;
+            local_endpoint_ = stream_->local_endpoint();
+            local_endpoint_address_ = local_endpoint_.address().to_string();
             endpoint_ = it->endpoint();
+            endpoint_address_ = endpoint_.address().to_string();
             spdlog::debug("{} connected to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
             log_prefix_ = fmt::format("[{}/{}] <{}:{}>", client_id_, id_, endpoint_.address().to_string(), endpoint_.port());
             deadline_timer_.expires_at(asio::steady_timer::time_point::max());
@@ -219,10 +323,10 @@ class http_session : public std::enable_shared_from_this<http_session>
             return;
         }
         if (deadline_timer_.expiry() <= asio::steady_timer::clock_type::now()) {
-            socket_.close();
+            stream_->close();
             deadline_timer_.expires_at(asio::steady_timer::time_point::max());
         }
-        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, this, std::placeholders::_1));
+        deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1));
     }
 
     void do_read()
@@ -230,12 +334,13 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (stopped_) {
             return;
         }
-        socket_.async_read_some(
+        stream_->async_read_some(
           asio::buffer(input_buffer_), [self = shared_from_this()](std::error_code ec, std::size_t bytes_transferred) {
-              if (self->stopped_) {
+              if (ec == asio::error::operation_aborted || self->stopped_) {
                   return;
               }
-              if (ec && ec != asio::error::operation_aborted) {
+              self->last_active_ = std::chrono::steady_clock::now();
+              if (ec) {
                   spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
                   return self->stop();
               }
@@ -244,12 +349,19 @@ class http_session : public std::enable_shared_from_this<http_session>
                   case http_parser::status::ok:
                       if (self->parser_.complete) {
                           if (!self->command_handlers_.empty()) {
-                              auto handler = self->command_handlers_.front();
-                              self->command_handlers_.pop_front();
+                              decltype(self->command_handlers_)::value_type handler{};
+                              {
+                                  std::scoped_lock lock(self->command_handlers_mutex_);
+                                  handler = self->command_handlers_.front();
+                                  self->command_handlers_.pop_front();
+                              }
+                              if (self->parser_.response.must_close_connection()) {
+                                  self->keep_alive_ = false;
+                              }
                               handler({}, std::move(self->parser_.response));
                           }
                           self->parser_.reset();
-                          return self->stop();
+                          return;
                       }
                       return self->do_read();
                   case http_parser::status::failure:
@@ -273,10 +385,11 @@ class http_session : public std::enable_shared_from_this<http_session>
         for (auto& buf : writing_buffer_) {
             buffers.emplace_back(asio::buffer(buf));
         }
-        asio::async_write(socket_, buffers, [self = shared_from_this()](std::error_code ec, std::size_t /* bytes_transferred */) {
-            if (self->stopped_) {
+        stream_->async_write(buffers, [self = shared_from_this()](std::error_code ec, std::size_t /* bytes_transferred */) {
+            if (ec == asio::error::operation_aborted || self->stopped_) {
                 return;
             }
+            self->last_active_ = std::chrono::steady_clock::now();
             if (ec) {
                 spdlog::error("{} IO error while writing to the socket: {}", self->log_prefix_, ec.message());
                 return self->stop();
@@ -289,33 +402,43 @@ class http_session : public std::enable_shared_from_this<http_session>
         });
     }
 
+    service_type type_;
     std::string client_id_;
     std::string id_;
     asio::io_context& ctx_;
     asio::ip::tcp::resolver resolver_;
-    asio::strand<asio::io_context::executor_type> strand_;
-    asio::ip::tcp::socket socket_;
+    std::unique_ptr<stream_impl> stream_;
     asio::steady_timer deadline_timer_;
+    asio::steady_timer idle_timer_;
 
-    std::string username_;
-    std::string password_;
+    cluster_credentials credentials_;
     std::string hostname_;
     std::string service_;
     std::string user_agent_;
 
     bool stopped_{ false };
     bool connected_{ false };
+    bool keep_alive_{ false };
 
     std::function<void()> on_stop_handler_{ nullptr };
 
     std::list<std::function<void(std::error_code, io::http_response&&)>> command_handlers_{};
+    std::mutex command_handlers_mutex_{};
+
     http_parser parser_{};
     std::array<std::uint8_t, 16384> input_buffer_{};
     std::vector<std::vector<std::uint8_t>> output_buffer_{};
     std::vector<std::vector<std::uint8_t>> writing_buffer_{};
     asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
-    asio::ip::tcp::resolver::results_type endpoints_;
+    std::string endpoint_address_{};     // cached string with endpoint address
+    asio::ip::tcp::endpoint local_endpoint_{};
+    std::string local_endpoint_address_{};
+    asio::ip::tcp::resolver::results_type endpoints_{};
 
     std::string log_prefix_{};
+    couchbase::http_context http_ctx_;
+
+    std::chrono::time_point<std::chrono::steady_clock> last_active_{};
+    diag::endpoint_state state_{ diag::endpoint_state::disconnected };
 };
 } // namespace couchbase::io