couchbase 3.0.0.alpha.3-universal-darwin-19 → 3.0.1-universal-darwin-19

data/ext/couchbase/capabilities.hxx

@@ -0,0 +1,117 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <spdlog/fmt/fmt.h>
+
+namespace couchbase
+{
+enum class bucket_capability {
+    couchapi,
+    xattr,
+    dcp,
+    cbhello,
+    touch,
+    cccp,
+    xdcr_checkpointing,
+    nodes_ext,
+    collections,
+    durable_write,
+    tombstoned_user_xattrs,
+};
+
+enum class cluster_capability {
+    n1ql_cost_based_optimizer,
+    n1ql_index_advisor,
+    n1ql_javascript_functions,
+    n1ql_inline_functions,
+    n1ql_enhanced_prepared_statements,
+};
+} // namespace couchbase
+
+template<>
+struct fmt::formatter<couchbase::bucket_capability> : formatter<std::string_view> {
+    template<typename FormatContext>
+    auto format(couchbase::bucket_capability type, FormatContext& ctx)
+    {
+        string_view name = "unknown";
+        switch (type) {
+            case couchbase::bucket_capability::couchapi:
+                name = "couchapi";
+                break;
+            case couchbase::bucket_capability::xattr:
+                name = "xattr";
+                break;
+            case couchbase::bucket_capability::dcp:
+                name = "dcp";
+                break;
+            case couchbase::bucket_capability::cbhello:
+                name = "cbhello";
+                break;
+            case couchbase::bucket_capability::touch:
+                name = "touch";
+                break;
+            case couchbase::bucket_capability::cccp:
+                name = "cccp";
+                break;
+            case couchbase::bucket_capability::xdcr_checkpointing:
+                name = "xdcr_checkpointing";
+                break;
+            case couchbase::bucket_capability::nodes_ext:
+                name = "nodes_ext";
+                break;
+            case couchbase::bucket_capability::collections:
+                name = "collections";
+                break;
+            case couchbase::bucket_capability::durable_write:
+                name = "durable_write";
+                break;
+            case couchbase::bucket_capability::tombstoned_user_xattrs:
+                name = "tombstoned_user_xattrs";
+                break;
+        }
+        return formatter<string_view>::format(name, ctx);
+    }
+};
+
+template<>
+struct fmt::formatter<couchbase::cluster_capability> : formatter<std::string_view> {
+    template<typename FormatContext>
+    auto format(couchbase::cluster_capability type, FormatContext& ctx)
+    {
+        string_view name = "unknown";
+        switch (type) {
+            case couchbase::cluster_capability::n1ql_cost_based_optimizer:
+                name = "n1ql_cost_based_optimizer";
+                break;
+            case couchbase::cluster_capability::n1ql_index_advisor:
+                name = "n1ql_index_advisor";
+                break;
+            case couchbase::cluster_capability::n1ql_javascript_functions:
+                name = "n1ql_javascript_functions";
+                break;
+            case couchbase::cluster_capability::n1ql_inline_functions:
+                name = "n1ql_inline_functions";
+                break;
+            case couchbase::cluster_capability::n1ql_enhanced_prepared_statements:
+                name = "n1ql_enhanced_prepared_statements";
+                break;
+        }
+        return formatter<string_view>::format(name, ctx);
+    }
+};

data/ext/couchbase/cbcrypto/cbcrypto.cc

@@ -88,7 +88,7 @@ hash(std::string_view key, std::string_view data, LPCWSTR algorithm, int flags)
         throw std::runtime_error("digest: BCryptHashData return: " + std::to_string(status));
     }
 
-    status = BCryptFinishHash(hHash, reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())), cbHash, 0);
+    status = BCryptFinishHash(hHash, reinterpret_cast<uint8_t*>(ret.data()), cbHash, 0);
 
     // Release resources
     BCryptCloseAlgorithmProvider(hAlg, 0);
@@ -256,7 +256,7 @@ AES_256_cbc(bool encrypt, std::string_view key, std::string_view iv, std::string
                                nullptr,
                                (PBYTE)civ.data(),
                                ULONG(civ.size()),
-                               reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                               reinterpret_cast<uint8_t*>(ret.data()),
                                ULONG(ret.size()),
                                &cbData,
                                BCRYPT_BLOCK_PADDING);
@@ -267,7 +267,7 @@ AES_256_cbc(bool encrypt, std::string_view key, std::string_view iv, std::string
                                nullptr,
                                (PBYTE)civ.data(),
                                ULONG(civ.size()),
-                               reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                               reinterpret_cast<uint8_t*>(ret.data()),
                                ULONG(ret.size()),
                                &cbData,
                                BCRYPT_BLOCK_PADDING);
@@ -309,7 +309,7 @@ HMAC_SHA1(std::string_view key, std::string_view data)
 {
     std::string ret;
     ret.resize(couchbase::crypto::SHA1_DIGEST_SIZE);
-    CCHmac(kCCHmacAlgSHA1, key.data(), key.size(), data.data(), data.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())));
+    CCHmac(kCCHmacAlgSHA1, key.data(), key.size(), data.data(), data.size(), reinterpret_cast<uint8_t*>(ret.data()));
     return ret;
 }
 
@@ -318,7 +318,7 @@ HMAC_SHA256(std::string_view key, std::string_view data)
 {
     std::string ret;
     ret.resize(couchbase::crypto::SHA256_DIGEST_SIZE);
-    CCHmac(kCCHmacAlgSHA256, key.data(), key.size(), data.data(), data.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())));
+    CCHmac(kCCHmacAlgSHA256, key.data(), key.size(), data.data(), data.size(), reinterpret_cast<uint8_t*>(ret.data()));
     return ret;
 }
 
@@ -327,7 +327,7 @@ HMAC_SHA512(std::string_view key, std::string_view data)
 {
     std::string ret;
     ret.resize(couchbase::crypto::SHA512_DIGEST_SIZE);
-    CCHmac(kCCHmacAlgSHA512, key.data(), key.size(), data.data(), data.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())));
+    CCHmac(kCCHmacAlgSHA512, key.data(), key.size(), data.data(), data.size(), reinterpret_cast<uint8_t*>(ret.data()));
     return ret;
 }
 
@@ -343,7 +343,7 @@ PBKDF2_HMAC_SHA1(const std::string& pass, std::string_view salt, unsigned int it
                                     salt.size(),
                                     kCCPRFHmacAlgSHA1,
                                     iterationCount,
-                                    reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                                    reinterpret_cast<uint8_t*>(ret.data()),
                                     ret.size());
 
     if (err != 0) {
@@ -364,7 +364,7 @@ PBKDF2_HMAC_SHA256(const std::string& pass, std::string_view salt, unsigned int
                                     salt.size(),
                                     kCCPRFHmacAlgSHA256,
                                     iterationCount,
-                                    reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                                    reinterpret_cast<uint8_t*>(ret.data()),
                                     ret.size());
     if (err != 0) {
         throw std::runtime_error("couchbase::crypto::PBKDF2_HMAC(SHA256): CCKeyDerivationPBKDF "
@@ -386,7 +386,7 @@ PBKDF2_HMAC_SHA512(const std::string& pass, std::string_view salt, unsigned int
                                     salt.size(),
                                     kCCPRFHmacAlgSHA512,
                                     iterationCount,
-                                    reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                                    reinterpret_cast<uint8_t*>(ret.data()),
                                     ret.size());
     if (err != 0) {
         throw std::runtime_error("couchbase::crypto::PBKDF2_HMAC(SHA512): CCKeyDerivationPBKDF "
@@ -401,7 +401,7 @@ digest_sha1(std::string_view data)
 {
     std::string ret;
     ret.resize(couchbase::crypto::SHA1_DIGEST_SIZE);
-    CC_SHA1(data.data(), data.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())));
+    CC_SHA1(data.data(), static_cast<CC_LONG>(data.size()), reinterpret_cast<uint8_t*>(ret.data()));
     return ret;
 }
 
@@ -410,7 +410,7 @@ digest_sha256(std::string_view data)
 {
     std::string ret;
     ret.resize(couchbase::crypto::SHA256_DIGEST_SIZE);
-    CC_SHA256(data.data(), data.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())));
+    CC_SHA256(data.data(), static_cast<CC_LONG>(data.size()), reinterpret_cast<uint8_t*>(ret.data()));
     return ret;
 }
 
@@ -419,7 +419,7 @@ digest_sha512(std::string_view data)
 {
     std::string ret;
     ret.resize(couchbase::crypto::SHA512_DIGEST_SIZE);
-    CC_SHA512(data.data(), data.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())));
+    CC_SHA512(data.data(), static_cast<CC_LONG>(data.size()), reinterpret_cast<uint8_t*>(ret.data()));
     return ret;
 }
 
@@ -470,7 +470,7 @@ encrypt(const couchbase::crypto::Cipher cipher, std::string_view key, std::strin
                           iv.data(),
                           data.data(),
                           data.size(),
-                          reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                          reinterpret_cast<uint8_t*>(ret.data()),
                           ret.size(),
                           &outputsize);
 
@@ -499,7 +499,7 @@ decrypt(const couchbase::crypto::Cipher cipher, std::string_view key, std::strin
                           iv.data(),
                           data.data(),
                           data.size(),
-                          reinterpret_cast<uint8_t*>(const_cast<char*>(ret.data())),
+                          reinterpret_cast<uint8_t*>(ret.data()),
                           ret.size(),
                           &outputsize);
 
@@ -718,7 +718,7 @@ encrypt(const couchbase::crypto::Cipher cipher, std::string_view key, std::strin
     }
 
     std::string ret;
-    ret.resize(data.size() + static_cast<std::size_t>(EVP_CIPHER_block_size(cip)));
+    ret.resize(data.size() + static_cast<std::string_view::size_type>(EVP_CIPHER_block_size(cip)));
     int len1 = int(ret.size());
 
     if (EVP_EncryptUpdate(

data/ext/couchbase/cluster.hxx

@@ -19,124 +19,83 @@
 
 #include <utility>
 #include <thread>
+#include <fstream>
+
+#include <asio/ssl.hpp>
 
 #include <io/mcbp_session.hxx>
-#include <io/session_manager.hxx>
+#include <io/http_session_manager.hxx>
+#include <io/http_command.hxx>
+#include <io/dns_client.hxx>
+#include <origin.hxx>
 #include <bucket.hxx>
 #include <operations.hxx>
 #include <operations/document_query.hxx>
 
+#include <diagnostics.hxx>
+
 namespace couchbase
 {
-struct origin {
-    std::string username;
-    std::string password;
-    std::string hostname;
-
-    [[nodiscard]] const std::string& get_username() const
-    {
-        return username;
-    }
-
-    [[nodiscard]] const std::string& get_password() const
-    {
-        return password;
-    }
-
-    [[nodiscard]] std::pair<std::string, std::string> get_address() const
-    {
-        return { hostname, "11210" };
-    }
-};
-
 class cluster
 {
   public:
     explicit cluster(asio::io_context& ctx)
-      : id_(uuid::random())
+      : id_(uuid::to_string(uuid::random()))
       , ctx_(ctx)
       , work_(asio::make_work_guard(ctx_))
-      , session_(std::make_shared<io::mcbp_session>(id_, ctx_))
-      , session_manager_(std::make_shared<io::session_manager>(id_, ctx_))
+      , tls_(asio::ssl::context::tls_client)
+      , session_manager_(std::make_shared<io::http_session_manager>(id_, ctx_, tls_))
+      , dns_config_(io::dns::dns_config::get())
+      , dns_client_(ctx_)
     {
     }
 
-    ~cluster()
-    {
-        work_.reset();
-    }
-
     template<typename Handler>
     void open(const couchbase::origin& origin, Handler&& handler)
     {
-        auto address = origin.get_address();
         origin_ = origin;
-        session_->bootstrap(address.first,
-                            address.second,
-                            origin.get_username(),
-                            origin.get_password(),
-                            [this, handler = std::forward<Handler>(handler)](std::error_code ec, configuration config) mutable {
-                                session_manager_->set_configuration(config);
-                                handler(ec);
-                            });
+        if (origin_.options().enable_dns_srv) {
+            return asio::post(asio::bind_executor(
+              ctx_, [this, handler = std::forward<Handler>(handler)]() mutable { return do_dns_srv(std::forward<Handler>(handler)); }));
+        }
+        do_open(std::forward<Handler>(handler));
     }
 
     template<typename Handler>
     void close(Handler&& handler)
     {
         asio::post(asio::bind_executor(ctx_, [this, handler = std::forward<Handler>(handler)]() {
-            session_->stop();
+            if (session_) {
+                session_->stop(io::retry_reason::do_not_retry);
+            }
             for (auto& bucket : buckets_) {
                 bucket.second->close();
             }
+            session_manager_->close();
             handler();
+            work_.reset();
         }));
     }
 
     template<typename Handler>
     void open_bucket(const std::string& bucket_name, Handler&& handler)
     {
-        if (!session_->has_config()) {
-            return handler(std::make_error_code(error::common_errc::invalid_argument));
+        auto ptr = buckets_.find(bucket_name);
+        if (ptr != buckets_.end()) {
+            return handler({});
         }
-        configuration config = session_->config();
-        auto& node = config.nodes.front();
-        auto new_session = std::make_shared<io::mcbp_session>(id_, ctx_, bucket_name);
-        new_session->bootstrap(
-          node.hostname,
-          std::to_string(*node.services_plain.key_value),
-          origin_.get_username(),
-          origin_.get_password(),
-          [this, name = bucket_name, new_session, h = std::forward<Handler>(handler)](std::error_code ec, configuration cfg) mutable {
-              if (!ec) {
-                  if (!session_->supports_gcccp()) {
-                      session_manager_->set_configuration(cfg);
-                  }
-                  auto b = std::make_shared<bucket>(ctx_, name, cfg);
-                  size_t this_index = new_session->index();
-                  b->set_node(this_index, new_session);
-                  if (cfg.nodes.size() > 1) {
-                      for (const auto& n : cfg.nodes) {
-                          if (n.index != this_index) {
-                              auto s = std::make_shared<io::mcbp_session>(id_, ctx_, name);
-                              b->set_node(n.index, s);
-                              s->bootstrap(n.hostname,
-                                           std::to_string(*n.services_plain.key_value),
-                                           origin_.get_username(),
-                                           origin_.get_password(),
-                                           [s, i = n.index, b](std::error_code err, configuration /*config*/) {
-                                               if (err) {
-                                                   spdlog::warn("unable to bootstrap node: {}", err.message());
-                                                   b->remove_node(i);
-                                               }
-                                           });
-                          }
-                      }
-                  }
-                  buckets_.emplace(name, std::move(b));
-              }
-              h(ec);
-          });
+        std::vector<protocol::hello_feature> known_features;
+        if (session_ && session_->has_config()) {
+            known_features = session_->supported_features();
+        }
+        auto b = std::make_shared<bucket>(id_, ctx_, tls_, bucket_name, origin_, known_features);
+        b->bootstrap([this, handler = std::forward<Handler>(handler)](std::error_code ec, const configuration& config) mutable {
+            if (!ec && !session_->supports_gcccp()) {
+                session_manager_->set_configuration(config, origin_.options());
+            }
+            handler(ec);
+        });
+        buckets_.emplace(bucket_name, b);
     }
 
     template<class Request, class Handler>
@@ -152,24 +111,157 @@ class cluster
     template<class Request, class Handler>
     void execute_http(Request request, Handler&& handler)
     {
-        auto session = session_manager_->check_out(Request::type, origin_.username, origin_.password);
+        auto session = session_manager_->check_out(Request::type, origin_.credentials());
         if (!session) {
             return handler(operations::make_response(std::make_error_code(error::common_errc::service_not_available), request, {}));
         }
-        auto cmd = std::make_shared<operations::command<Request>>(ctx_, std::move(request));
+        auto cmd = std::make_shared<operations::http_command<Request>>(ctx_, request);
         cmd->send_to(session, [this, session, handler = std::forward<Handler>(handler)](typename Request::response_type resp) mutable {
             handler(std::move(resp));
             session_manager_->check_in(Request::type, session);
         });
     }
 
+    template<typename Handler>
+    void diagnostics(std::optional<std::string> report_id, Handler&& handler)
+    {
+        if (!report_id) {
+            report_id = std::make_optional(uuid::to_string(uuid::random()));
+        }
+        asio::post(asio::bind_executor(ctx_, [this, report_id, handler = std::forward<Handler>(handler)]() mutable {
+            diag::diagnostics_result res{
+                report_id.value(),
+                fmt::format("ruby/{}.{}.{}/{}", BACKEND_VERSION_MAJOR, BACKEND_VERSION_MINOR, BACKEND_VERSION_PATCH, BACKEND_GIT_REVISION)
+            };
+            if (session_) {
+                res.services[service_type::kv].emplace_back(session_->diag_info());
+            }
+            for (const auto& bucket : buckets_) {
+                bucket.second->export_diag_info(res);
+            }
+            session_manager_->export_diag_info(res);
+            handler(std::move(res));
+        }));
+    }
+
   private:
-    uuid::uuid_t id_;
+    template<typename Handler>
+    void do_dns_srv(Handler&& handler)
+    {
+        std::string hostname;
+        std::string service;
+        std::tie(hostname, service) = origin_.next_address();
+        service = origin_.options().enable_tls ? "_couchbases" : "_couchbase";
+        dns_client_.query_srv(
+          hostname,
+          service,
+          [hostname, this, handler = std::forward<Handler>(handler)](couchbase::io::dns::dns_client::dns_srv_response resp) mutable {
+              if (resp.ec) {
+                  spdlog::warn("failed to fetch DNS SRV records for \"{}\" ({}), assuming that cluster is listening this address",
+                               hostname,
+                               resp.ec.message());
+              } else if (resp.targets.empty()) {
+                  spdlog::warn("DNS SRV query returned 0 records for \"{}\", assuming that cluster is listening this address", hostname);
+              } else {
+                  origin::node_list nodes;
+                  nodes.reserve(resp.targets.size());
+                  for (const auto& address : resp.targets) {
+                      origin::node_entry node;
+                      node.first = address.hostname;
+                      node.second = std::to_string(address.port);
+                      nodes.emplace_back(node);
+                  }
+                  origin_.set_nodes(nodes);
+                  spdlog::info("replace list of bootstrap nodes with addresses from DNS SRV of \"{}\": [{}]",
+                               hostname,
+                               fmt::join(origin_.get_nodes(), ", "));
+              }
+              return do_open(std::forward<Handler>(handler));
+          });
+    }
+
+    template<typename Handler>
+    void do_open(Handler&& handler)
+    {
+        if (origin_.options().enable_tls) {
+            tls_.set_options(asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv2 | asio::ssl::context::no_sslv3);
+            if (!origin_.options().trust_certificate.empty()) {
+                std::error_code ec{};
+                spdlog::debug(R"([{}]: use TLS certificate chain: "{}")", id_, origin_.options().trust_certificate);
+                tls_.use_certificate_chain_file(origin_.options().trust_certificate, ec);
+                if (ec) {
+                    spdlog::error(
+                      "[{}]: unable to load certificate chain \"{}\": {}", id_, origin_.options().trust_certificate, ec.message());
+                    return handler(ec);
+                }
+            }
+#ifdef TLS_KEY_LOG_FILE
+            SSL_CTX_set_keylog_callback(tls_.native_handle(), [](const SSL* /* ssl */, const char* line) {
+                std::ofstream keylog(TLS_KEY_LOG_FILE, std::ios::out | std::ios::app | std::ios::binary);
+                keylog << std::string_view(line) << std::endl;
+            });
+            spdlog::critical("TLS_KEY_LOG_FILE was set to \"{}\" during build, all TLS keys will be logged for network analysis "
+                             "(https://wiki.wireshark.org/TLS). DO NOT USE THIS BUILD IN PRODUCTION",
+                             TLS_KEY_LOG_FILE);
+#endif
+            if (origin_.credentials().uses_certificate()) {
+                std::error_code ec{};
+                spdlog::debug(R"([{}]: use TLS certificate: "{}")", id_, origin_.certificate_path());
+                tls_.use_certificate_file(origin_.certificate_path(), asio::ssl::context::file_format::pem, ec);
+                if (ec) {
+                    spdlog::error("[{}]: unable to load certificate \"{}\": {}", id_, origin_.certificate_path(), ec.message());
+                    return handler(ec);
+                }
+                spdlog::debug(R"([{}]: use TLS private key: "{}")", id_, origin_.key_path());
+                tls_.use_private_key_file(origin_.key_path(), asio::ssl::context::file_format::pem, ec);
+                if (ec) {
+                    spdlog::error("[{}]: unable to load private key \"{}\": {}", id_, origin_.key_path(), ec.message());
+                    return handler(ec);
+                }
+            }
+            session_ = std::make_shared<io::mcbp_session>(id_, ctx_, tls_, origin_);
+        } else {
+            session_ = std::make_shared<io::mcbp_session>(id_, ctx_, origin_);
+        }
+        session_->bootstrap([this, handler = std::forward<Handler>(handler)](std::error_code ec, const configuration& config) mutable {
+            if (!ec) {
+                if (origin_.options().network == "auto") {
+                    origin_.options().network = config.select_network(session_->bootstrap_hostname());
+                    spdlog::info(R"({} detected network is "{}")", session_->log_prefix(), origin_.options().network);
+                }
+                if (origin_.options().network != "default") {
+                    origin::node_list nodes;
+                    nodes.reserve(config.nodes.size());
+                    for (const auto& address : config.nodes) {
+                        auto port = address.port_or(origin_.options().network, service_type::kv, origin_.options().enable_tls, 0);
+                        if (port == 0) {
+                            continue;
+                        }
+                        origin::node_entry node;
+                        node.first = address.hostname_for(origin_.options().network);
+                        node.second = std::to_string(port);
+                        nodes.emplace_back(node);
+                    }
+                    origin_.set_nodes(nodes);
+                    spdlog::info("replace list of bootstrap nodes with addresses of alternative network \"{}\": [{}]",
+                                 origin_.options().network,
+                                 fmt::join(origin_.get_nodes(), ","));
+                }
+                session_manager_->set_configuration(config, origin_.options());
+            }
+            handler(ec);
+        });
+    }
+
+    std::string id_;
     asio::io_context& ctx_;
     asio::executor_work_guard<asio::io_context::executor_type> work_;
-    std::shared_ptr<io::mcbp_session> session_;
-    std::shared_ptr<io::session_manager> session_manager_;
-    std::map<std::string, std::shared_ptr<bucket>> buckets_;
-    origin origin_;
+    asio::ssl::context tls_;
+    std::shared_ptr<io::http_session_manager> session_manager_;
+    io::dns::dns_config& dns_config_;
+    couchbase::io::dns::dns_client dns_client_;
+    std::shared_ptr<io::mcbp_session> session_{};
+    std::map<std::string, std::shared_ptr<bucket>> buckets_{};
+    couchbase::origin origin_{};
 };
 } // namespace couchbase