couchbase 3.0.0 → 3.0.1

data/ext/couchbase/diagnostics.hxx

@@ -0,0 +1,251 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <map>
+#include <vector>
+#include <chrono>
+
+#include <spdlog/fmt/fmt.h>
+
+#include <service_type.hxx>
+
+namespace couchbase::diag
+{
+enum class cluster_state {
+    /** all nodes and their sockets are reachable */
+    online,
+    /** at least one socket per service is reachable */
+    degraded,
+    /** not even one socket per service is reachable */
+    offline,
+};
+
+enum class endpoint_state {
+    /** the endpoint is not reachable */
+    disconnected,
+    /** currently connecting (includes auth, ...) */
+    connecting,
+    /** connected and ready */
+    connected,
+    /** disconnecting (after being connected) */
+    disconnecting,
+};
+
+struct endpoint_diag_info {
+    service_type type;
+    std::string id;
+    std::optional<std::chrono::microseconds> last_activity{};
+    std::string remote;
+    std::string local;
+    endpoint_state state;
+    /** serialized as "namespace" */
+    std::optional<std::string> bucket{};
+    std::optional<std::string> details{};
+};
+
+struct diagnostics_result {
+    std::string id;
+    std::string sdk;
+    std::map<service_type, std::vector<endpoint_diag_info>> services{};
+
+    const int version{ 2 };
+};
+
+enum class ping_state {
+    ok,
+    timeout,
+    error,
+};
+
+struct endpoint_ping_info {
+    service_type type;
+    std::string id;
+    std::chrono::microseconds latency;
+    std::string remote;
+    std::string local;
+    ping_state state;
+    /** serialized as "namespace" */
+    std::optional<std::string> bucket{};
+    /** if ping state is error, contains error message */
+    std::optional<std::string> error{};
+};
+
+struct ping_result {
+    std::string id;
+    std::string sdk;
+    std::uint64_t config_rev;
+    std::map<service_type, std::vector<endpoint_ping_info>> services{};
+
+    const int version{ 2 };
+};
+} // namespace couchbase::diag
+
+template<>
+struct fmt::formatter<couchbase::diag::cluster_state> : formatter<std::string_view> {
+    template<typename FormatContext>
+    auto format(couchbase::diag::cluster_state state, FormatContext& ctx)
+    {
+        string_view name = "unknown";
+        switch (state) {
+            case couchbase::diag::cluster_state::online:
+                name = "online";
+                break;
+
+            case couchbase::diag::cluster_state::degraded:
+                name = "degraded";
+                break;
+
+            case couchbase::diag::cluster_state::offline:
+                name = "offline";
+                break;
+        }
+        return formatter<string_view>::format(name, ctx);
+    }
+};
+
+template<>
+struct fmt::formatter<couchbase::diag::endpoint_state> : formatter<std::string_view> {
+    template<typename FormatContext>
+    auto format(couchbase::diag::endpoint_state state, FormatContext& ctx)
+    {
+        string_view name = "unknown";
+        switch (state) {
+            case couchbase::diag::endpoint_state::disconnected:
+                name = "disconnected";
+                break;
+
+            case couchbase::diag::endpoint_state::connecting:
+                name = "connecting";
+                break;
+
+            case couchbase::diag::endpoint_state::connected:
+                name = "connected";
+                break;
+
+            case couchbase::diag::endpoint_state::disconnecting:
+                name = "disconnecting";
+                break;
+        }
+        return formatter<string_view>::format(name, ctx);
+    }
+};
+
+template<>
+struct fmt::formatter<couchbase::diag::ping_state> : formatter<std::string_view> {
+    template<typename FormatContext>
+    auto format(couchbase::diag::ping_state state, FormatContext& ctx)
+    {
+        string_view name = "unknown";
+        switch (state) {
+            case couchbase::diag::ping_state::ok:
+                name = "ok";
+                break;
+
+            case couchbase::diag::ping_state::timeout:
+                name = "timeout";
+                break;
+
+            case couchbase::diag::ping_state::error:
+                name = "error";
+                break;
+        }
+        return formatter<string_view>::format(name, ctx);
+    }
+};
+
+namespace tao::json
+{
+template<>
+struct traits<couchbase::diag::diagnostics_result> {
+    template<template<typename...> class Traits>
+    static void assign(tao::json::basic_value<Traits>& v, const couchbase::diag::diagnostics_result& r)
+    {
+        tao::json::value services = tao::json::empty_object;
+        for (const auto& entry : r.services) {
+            tao::json::value service{};
+            for (const auto& endpoint : entry.second) {
+                tao::json::value e = tao::json::empty_object;
+                if (endpoint.last_activity) {
+                    e["last_activity_us"] = endpoint.last_activity->count();
+                }
+                e["remote"] = endpoint.remote;
+                e["local"] = endpoint.local;
+                e["id"] = endpoint.id;
+                e["state"] = fmt::format("{}", endpoint.state);
+                if (endpoint.bucket) {
+                    e["namespace"] = endpoint.bucket.value();
+                }
+                if (endpoint.details) {
+                    e["details"] = endpoint.details.value();
+                }
+                service.push_back(e);
+            }
+            services[fmt::format("{}", entry.first)] = service;
+        }
+
+        v = {
+            { "version", r.version },
+            { "id", r.id },
+            { "sdk", r.sdk },
+            { "services", services },
+        };
+    }
+};
+} // namespace tao::json
+
+namespace tao::json
+{
+template<>
+struct traits<couchbase::diag::ping_result> {
+    template<template<typename...> class Traits>
+    static void assign(tao::json::basic_value<Traits>& v, const couchbase::diag::ping_result& r)
+    {
+        tao::json::value services{};
+        for (const auto& entry : r.services) {
+            tao::json::value service{};
+            for (const auto& endpoint : entry.second) {
+                tao::json::value e{};
+                e["latency_us"] = endpoint.latency.count();
+                e["remote"] = endpoint.remote;
+                e["local"] = endpoint.local;
+                e["id"] = endpoint.id;
+                e["state"] = fmt::format("{}", endpoint.state);
+                if (endpoint.bucket) {
+                    e["namespace"] = endpoint.bucket.value();
+                }
+                if (endpoint.state == couchbase::diag::ping_state::error && endpoint.error) {
+                    e["error"] = endpoint.error.value();
+                }
+                service.push_back(e);
+            }
+            services[fmt::format("{}", entry.first)] = service;
+        }
+
+        v = {
+            { "version", r.version },
+            { "id", r.id },
+            { "sdk", r.sdk },
+            /* the highest known configuration revision */
+            { "config_rev", r.config_rev },
+            { "services", services },
+        };
+    }
+};
+} // namespace tao::json

data/ext/couchbase/document_id.hxx

@@ -23,7 +23,9 @@ struct document_id {
     std::string bucket;
     std::string collection;
     std::string key;
-    std::optional<std::uint32_t> collection_uid; // filled with resolved UID during request lifetime
+    std::optional<std::uint32_t> collection_uid{}; // filled with resolved UID during request lifetime
+    bool use_collections{ true };
+    bool use_any_session{ false };
 };
 } // namespace couchbase
 

data/ext/couchbase/errors.hxx

@@ -249,12 +249,14 @@ enum class analytics_errc {
 /// Errors related to Search service (CBFT)
 enum class search_errc {
     index_not_ready = 400,
+
+    consistency_mismatch = 401,
 };
 
 /// Errors related to Views service (CAPI)
 enum class view_errc {
     /// Http status code 404
-    /// Reason or error contains “not_found”
+    /// Reason or error contains "not_found"
     view_not_found = 500,
 
     /// Raised on the Management APIs only when
@@ -301,6 +303,9 @@ enum class network_errc {
 
     /// Unexpected protocol state or input
     protocol_error,
+
+    /// Configuration is not available for some reason
+    configuration_not_available,
 };
 
 namespace detail
@@ -472,6 +477,8 @@ struct search_error_category : std::error_category {
         switch (static_cast<search_errc>(ev)) {
             case search_errc::index_not_ready:
                 return "index_not_ready";
+            case search_errc::consistency_mismatch:
+                return "consistency_mismatch";
         }
         return "FIXME: unknown error code in search category (recompile with newer library)";
     }
@@ -596,6 +603,8 @@ struct network_error_category : std::error_category {
                 return "handshake_failure";
             case network_errc::protocol_error:
                 return "protocol_error";
+            case network_errc::configuration_not_available:
+                return "configuration_not_available";
         }
         return "FIXME: unknown error code in network category (recompile with newer library)";
     }

data/ext/couchbase/io/http_command.hxx

@@ -42,17 +42,17 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
     void send_to(std::shared_ptr<io::http_session> session, Handler&& handler)
     {
         encoded.type = Request::type;
-        request.encode_to(encoded);
+        request.encode_to(encoded, session->http_context());
         encoded.headers["client-context-id"] = request.client_context_id;
         auto log_prefix = session->log_prefix();
-        spdlog::trace("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms",
+        spdlog::trace(R"({} HTTP request: {}, method={}, path="{}", client_context_id="{}", timeout={}ms)",
                       log_prefix,
                       encoded.type,
                       encoded.method,
                       encoded.path,
                       request.client_context_id,
                       request.timeout.count());
-        SPDLOG_TRACE("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms{:a}",
+        SPDLOG_TRACE(R"({} HTTP request: {}, method={}, path="{}", client_context_id="{}", timeout={}ms{:a})",
                      log_prefix,
                      encoded.type,
                      encoded.method,
@@ -61,22 +61,26 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
                      request.timeout.count(),
                      spdlog::to_hex(encoded.body));
         session->write_and_subscribe(encoded,
-                                     [self = this->shared_from_this(), log_prefix, handler = std::forward<Handler>(handler)](
+                                     [self = this->shared_from_this(), log_prefix, session, handler = std::forward<Handler>(handler)](
                                        std::error_code ec, io::http_response&& msg) mutable {
                                          self->deadline.cancel();
                                          encoded_response_type resp(msg);
-                                         spdlog::trace("{} HTTP response: {}, client_context_id={}, status={}",
+                                         spdlog::trace(R"({} HTTP response: {}, client_context_id="{}", status={})",
                                                        log_prefix,
                                                        self->request.type,
                                                        self->request.client_context_id,
                                                        resp.status_code);
-                                         SPDLOG_TRACE("{} HTTP response: {}, client_context_id={}, status={}{:a}",
+                                         SPDLOG_TRACE(R"({} HTTP response: {}, client_context_id="{}", status={}{:a})",
                                                       log_prefix,
                                                       self->request.type,
                                                       self->request.client_context_id,
                                                       resp.status_code,
                                                       spdlog::to_hex(resp.body));
-                                         handler(make_response(ec, self->request, resp));
+                                         try {
+                                             handler(make_response(ec, self->request, resp));
+                                         } catch (priv::retry_http_request) {
+                                             self->send_to(session, std::forward<Handler>(handler));
+                                         }
                                      });
         deadline.expires_after(request.timeout);
         deadline.async_wait([session](std::error_code ec) {

data/ext/couchbase/io/http_context.hxx

@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <configuration.hxx>
+#include <io/query_cache.hxx>
+
+namespace couchbase
+{
+struct http_context {
+    const configuration& config;
+    const cluster_options& options;
+    query_cache& cache;
+};
+
+namespace priv
+{
+class retry_http_request
+{
+};
+} // namespace priv
+} // namespace couchbase

data/ext/couchbase/io/http_message.hxx

@@ -34,5 +34,14 @@ struct http_response {
     std::string status_message;
     std::map<std::string, std::string> headers;
     std::string body;
+
+    [[nodiscard]] bool must_close_connection() const
+    {
+        const auto it = headers.find("connection");
+        if (it != headers.end()) {
+            return it->second == "close";
+        }
+        return false;
+    }
 };
 } // namespace couchbase::io

data/ext/couchbase/io/http_parser.hxx

@@ -19,6 +19,7 @@
 
 #include <http_parser.h>
 #include <io/http_message.hxx>
+#include <algorithm>
 
 namespace couchbase::io
 {
@@ -106,6 +107,7 @@ struct http_parser {
     int on_header_field(const char* at, std::size_t length)
     {
         header_field.assign(at, length);
+        std::transform(header_field.begin(), header_field.end(), header_field.begin(), [](unsigned char c) { return std::tolower(c); });
         return 0;
     }
 

data/ext/couchbase/io/http_session.hxx

@@ -32,6 +32,7 @@
 
 #include <io/http_parser.hxx>
 #include <io/http_message.hxx>
+#include <io/http_context.hxx>
 #include <platform/base64.h>
 #include <timeout_defaults.hxx>
 
@@ -41,17 +42,21 @@ namespace couchbase::io
 class http_session : public std::enable_shared_from_this<http_session>
 {
   public:
-    http_session(const std::string& client_id,
+    http_session(service_type type,
+                 const std::string& client_id,
                  asio::io_context& ctx,
                  const cluster_credentials& credentials,
                  const std::string& hostname,
-                 const std::string& service)
-      : client_id_(client_id)
+                 const std::string& service,
+                 http_context http_ctx)
+      : type_(type)
+      , client_id_(client_id)
       , id_(uuid::to_string(uuid::random()))
       , ctx_(ctx)
       , resolver_(ctx_)
       , stream_(std::make_unique<plain_stream_impl>(ctx_))
       , deadline_timer_(ctx_)
+      , idle_timer_(ctx_)
       , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
@@ -64,21 +69,26 @@ class http_session : public std::enable_shared_from_this<http_session>
                                 id_,
                                 BACKEND_SYSTEM))
       , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
+      , http_ctx_(std::move(http_ctx))
     {
     }
 
-    http_session(const std::string& client_id,
+    http_session(service_type type,
+                 const std::string& client_id,
                  asio::io_context& ctx,
                  asio::ssl::context& tls,
                  const cluster_credentials& credentials,
                  const std::string& hostname,
-                 const std::string& service)
-      : client_id_(client_id)
+                 const std::string& service,
+                 http_context http_ctx)
+      : type_(type)
+      , client_id_(client_id)
       , id_(uuid::to_string(uuid::random()))
       , ctx_(ctx)
       , resolver_(ctx_)
       , stream_(std::make_unique<tls_stream_impl>(ctx_, tls))
       , deadline_timer_(ctx_)
+      , idle_timer_(ctx_)
       , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
@@ -91,6 +101,7 @@ class http_session : public std::enable_shared_from_this<http_session>
                                 id_,
                                 BACKEND_SYSTEM))
       , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
+      , http_ctx_(std::move(http_ctx))
     {
     }
 
@@ -99,8 +110,26 @@ class http_session : public std::enable_shared_from_this<http_session>
         stop();
     }
 
+    [[nodiscard]] couchbase::http_context& http_context()
+    {
+        return http_ctx_;
+    }
+
+    [[nodiscard]] diag::endpoint_diag_info diag_info() const
+    {
+        return { type_,
+                 id_,
+                 last_active_.time_since_epoch().count() == 0 ? std::nullopt
+                                                              : std::make_optional(std::chrono::duration_cast<std::chrono::microseconds>(
+                                                                  std::chrono::steady_clock::now() - last_active_)),
+                 fmt::format("{}:{}", endpoint_address_, endpoint_.port()),
+                 fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port()),
+                 state_ };
+    }
+
     void start()
     {
+        state_ = diag::endpoint_state::connecting;
         resolver_.async_resolve(
           hostname_, service_, std::bind(&http_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
     }
@@ -127,21 +156,30 @@ class http_session : public std::enable_shared_from_this<http_session>
 
     void stop()
     {
+        if (stopped_) {
+            return;
+        }
         stopped_ = true;
+        state_ = diag::endpoint_state::disconnecting;
         if (stream_->is_open()) {
             stream_->close();
         }
         deadline_timer_.cancel();
+        idle_timer_.cancel();
 
-        for (auto& handler : command_handlers_) {
-            handler(std::make_error_code(error::common_errc::ambiguous_timeout), {});
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            for (auto& handler : command_handlers_) {
+                handler(std::make_error_code(error::common_errc::ambiguous_timeout), {});
+            }
+            command_handlers_.clear();
         }
-        command_handlers_.clear();
 
         if (on_stop_handler_) {
             on_stop_handler_();
             on_stop_handler_ = nullptr;
         }
+        state_ = diag::endpoint_state::disconnected;
     }
 
     bool keep_alive()
@@ -186,6 +224,9 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (stopped_) {
             return;
         }
+        if (request.headers["connection"] == "keep-alive") {
+            keep_alive_ = true;
+        }
         request.headers["user-agent"] = user_agent_;
         request.headers["authorization"] =
           fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", credentials_.username, credentials_.password)));
@@ -198,10 +239,29 @@ class http_session : public std::enable_shared_from_this<http_session>
         }
         write("\r\n");
         write(request.body);
-        command_handlers_.push_back(std::move(handler));
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            command_handlers_.push_back(std::move(handler));
+        }
         flush();
     }
 
+    void set_idle(std::chrono::milliseconds timeout)
+    {
+        idle_timer_.expires_after(timeout);
+        return idle_timer_.async_wait([self = shared_from_this()](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            self->stop();
+        });
+    }
+
+    void reset_idle()
+    {
+        idle_timer_.cancel();
+    }
+
   private:
     void on_resolve(std::error_code ec, const asio::ip::tcp::resolver::results_type& endpoints)
     {
@@ -209,6 +269,7 @@ class http_session : public std::enable_shared_from_this<http_session>
             spdlog::error("{} error on resolve: {}", log_prefix_, ec.message());
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         endpoints_ = endpoints;
         do_connect(endpoints_.begin());
         deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1));
@@ -231,13 +292,18 @@ class http_session : public std::enable_shared_from_this<http_session>
         if (stopped_) {
             return;
         }
+        last_active_ = std::chrono::steady_clock::now();
         if (!stream_->is_open() || ec) {
             spdlog::warn(
               "{} unable to connect to {}:{}: {}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port(), ec.message());
             do_connect(++it);
         } else {
+            state_ = diag::endpoint_state::connected;
             connected_ = true;
+            local_endpoint_ = stream_->local_endpoint();
+            local_endpoint_address_ = local_endpoint_.address().to_string();
             endpoint_ = it->endpoint();
+            endpoint_address_ = endpoint_.address().to_string();
             spdlog::debug("{} connected to {}:{}", log_prefix_, it->endpoint().address().to_string(), it->endpoint().port());
             log_prefix_ = fmt::format("[{}/{}] <{}:{}>", client_id_, id_, endpoint_.address().to_string(), endpoint_.port());
             deadline_timer_.expires_at(asio::steady_timer::time_point::max());
@@ -271,6 +337,7 @@ class http_session : public std::enable_shared_from_this<http_session>
               if (ec == asio::error::operation_aborted || self->stopped_) {
                   return;
               }
+              self->last_active_ = std::chrono::steady_clock::now();
               if (ec) {
                   spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
                   return self->stop();
@@ -280,12 +347,19 @@ class http_session : public std::enable_shared_from_this<http_session>
                   case http_parser::status::ok:
                       if (self->parser_.complete) {
                           if (!self->command_handlers_.empty()) {
-                              auto handler = self->command_handlers_.front();
-                              self->command_handlers_.pop_front();
+                              decltype(self->command_handlers_)::value_type handler{};
+                              {
+                                  std::scoped_lock lock(self->command_handlers_mutex_);
+                                  handler = self->command_handlers_.front();
+                                  self->command_handlers_.pop_front();
+                              }
+                              if (self->parser_.response.must_close_connection()) {
+                                  self->keep_alive_ = false;
+                              }
                               handler({}, std::move(self->parser_.response));
                           }
                           self->parser_.reset();
-                          return self->stop();
+                          return;
                       }
                       return self->do_read();
                   case http_parser::status::failure:
@@ -313,6 +387,7 @@ class http_session : public std::enable_shared_from_this<http_session>
             if (ec == asio::error::operation_aborted || self->stopped_) {
                 return;
             }
+            self->last_active_ = std::chrono::steady_clock::now();
             if (ec) {
                 spdlog::error("{} IO error while writing to the socket: {}", self->log_prefix_, ec.message());
                 return self->stop();
@@ -325,12 +400,14 @@ class http_session : public std::enable_shared_from_this<http_session>
         });
     }
 
+    service_type type_;
     std::string client_id_;
     std::string id_;
     asio::io_context& ctx_;
     asio::ip::tcp::resolver resolver_;
     std::unique_ptr<stream_impl> stream_;
     asio::steady_timer deadline_timer_;
+    asio::steady_timer idle_timer_;
 
     cluster_credentials credentials_;
     std::string hostname_;
@@ -344,13 +421,22 @@ class http_session : public std::enable_shared_from_this<http_session>
     std::function<void()> on_stop_handler_{ nullptr };
 
     std::list<std::function<void(std::error_code, io::http_response&&)>> command_handlers_{};
+    std::mutex command_handlers_mutex_{};
+
     http_parser parser_{};
     std::array<std::uint8_t, 16384> input_buffer_{};
     std::vector<std::vector<std::uint8_t>> output_buffer_{};
     std::vector<std::vector<std::uint8_t>> writing_buffer_{};
     asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
-    asio::ip::tcp::resolver::results_type endpoints_;
+    std::string endpoint_address_{};     // cached string with endpoint address
+    asio::ip::tcp::endpoint local_endpoint_{};
+    std::string local_endpoint_address_{};
+    asio::ip::tcp::resolver::results_type endpoints_{};
 
     std::string log_prefix_{};
+    couchbase::http_context http_ctx_;
+
+    std::chrono::time_point<std::chrono::steady_clock> last_active_{};
+    diag::endpoint_state state_{ diag::endpoint_state::disconnected };
 };
 } // namespace couchbase::io