couchbase 3.0.0.alpha.2-universal-darwin-19 → 3.0.0-universal-darwin-19

data/ext/.clang-tidy

@@ -17,4 +17,6 @@ Checks: |-
   -cppcoreguidelines-avoid-magic-numbers,
   -cppcoreguidelines-pro-bounds-pointer-arithmetic,
   -modernize-use-trailing-return-type,
+  -llvmlibc-callee-namespace,
+  -abseil-*,
   -fuchsia-*

data/ext/.idea/misc.xml

@@ -1,4 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="CMakeWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
+  <component name="CidrRootsConfiguration">
+    <sourceRoots>
+      <file path="$PROJECT_DIR$/couchbase" />
+    </sourceRoots>
+    <libraryRoots>
+      <file path="$PROJECT_DIR$/third_party" />
+    </libraryRoots>
+    <excludeRoots>
+      <file path="$PROJECT_DIR$/cmake-build-debug" />
+      <file path="$PROJECT_DIR$/cmake-build-debug-system-gcc" />
+    </excludeRoots>
+  </component>
 </project>

data/ext/CMakeLists.txt

@@ -8,6 +8,36 @@ if(CCACHE)
     set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE})
 endif()
 
+include(CMakePushCheckState)
+include(CheckSymbolExists)
+
+cmake_push_check_state(RESET)
+find_library(EXECINFO_LIBRARY NAMES execinfo)
+if(EXECINFO_LIBRARY)
+    set(CMAKE_REQUIRED_LIBRARIES "${EXECINFO_LIBRARY}")
+    list(APPEND PLATFORM_LIBRARIES "${EXECINFO_LIBRARY}")
+endif(EXECINFO_LIBRARY)
+check_symbol_exists(backtrace execinfo.h HAVE_BACKTRACE)
+cmake_pop_check_state()
+
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_DEFINITIONS "-D_GNU_SOURCE")
+find_library(DL_LIBRARY NAMES dl)
+if(DL_LIBRARY)
+    set(CMAKE_REQUIRED_LIBRARIES "${DL_LIBRARY}")
+    list(APPEND PLATFORM_LIBRARIES "${DL_LIBRARY}")
+endif(DL_LIBRARY)
+check_symbol_exists(dladdr dlfcn.h HAVE_DLADDR)
+cmake_pop_check_state()
+
+if(HAVE_BACKTRACE)
+    add_definitions(-DHAVE_BACKTRACE=1)
+endif()
+
+if(HAVE_DLADDR)
+    add_definitions(-DHAVE_DLADDR=1)
+endif()
+
 add_subdirectory(third_party/gsl)
 add_subdirectory(third_party/json)
 add_subdirectory(third_party/spdlog)
@@ -29,7 +59,7 @@ if(MSVC)
     target_compile_options(project_warnings INTERFACE /W4 /WX "/permissive-")
 else()
     option(ONLY_COVERAGE "Build only tests necessary for coverage" FALSE)
-    option(LIBCPP "Build with libc++" FALSE)
+    option(STATIC_STDLIB "Statically link C++ standard library" FALSE)
     option(ENABLE_COVERAGE "Enable coverage reporting for gcc/clang" FALSE)
     option(ENABLE_FUZZERS "Enable fuzz testing tools" FALSE)
 
@@ -52,6 +82,11 @@ else()
         target_link_libraries(project_options INTERFACE -fsanitize=thread)
     endif()
 
+    if(STATIC_STDLIB)
+        target_compile_options(project_options INTERFACE -static-libgcc -static-libstdc++)
+        target_link_options(project_options INTERFACE -static-libgcc -static-libstdc++)
+    endif()
+
     target_compile_options(
         project_warnings
         INTERFACE -Wall
@@ -85,8 +120,17 @@ else()
                       -Wuseless-cast # warn if you perform a cast to the same type
         )
     endif()
+
+    if(HAVE_BACKTRACE OR HAVE_DLADDR)
+        target_compile_options(project_options INTERFACE -ggdb3)
+        target_link_libraries(project_options INTERFACE -rdynamic)
+        add_definitions(-D_GNU_SOURCE=1)
+    endif()
 endif()
 
+# Read more at https://wiki.wireshark.org/TLS
+option(TLS_KEY_LOG_FILE "Path to file to write per-session secrets (Useful for Wireshark SSL/TLS dissection)")
+
 include(FindOpenSSL)
 message(STATUS "OPENSSL_VERSION: ${OPENSSL_VERSION}")
 message(STATUS "OPENSSL_INCLUDEDIRS: ${OPENSSL_INCLUDE_DIR}")
@@ -94,8 +138,10 @@ message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")
 
 include_directories(${CMAKE_SOURCE_DIR}/couchbase)
 
-add_library(platform OBJECT couchbase/platform/string_hex.cc couchbase/platform/uuid.cc couchbase/platform/random.cc
-                            couchbase/platform/base64.cc)
+add_library(
+    platform OBJECT couchbase/platform/string_hex.cc couchbase/platform/uuid.cc couchbase/platform/random.cc
+                    couchbase/platform/base64.cc couchbase/platform/backtrace.c couchbase/platform/terminate_handler.cc)
+target_include_directories(platform PRIVATE ${PROJECT_BINARY_DIR}/generated)
 
 add_library(cbcrypto OBJECT couchbase/cbcrypto/cbcrypto.cc)
 
@@ -129,6 +175,7 @@ endif()
 
 string(TIMESTAMP BACKEND_BUILD_TIMESTAMP "%Y-%m-%d %H:%M:%S" UTC)
 configure_file(${PROJECT_SOURCE_DIR}/build_version.hxx.in ${PROJECT_BINARY_DIR}/generated/build_version.hxx @ONLY)
+configure_file(${PROJECT_SOURCE_DIR}/build_config.hxx.in ${PROJECT_BINARY_DIR}/generated/build_config.hxx @ONLY)
 add_library(couchbase SHARED couchbase/couchbase.cxx)
 target_include_directories(couchbase PRIVATE ${PROJECT_BINARY_DIR}/generated)
 target_link_libraries(

data/ext/build_config.hxx.in

@@ -0,0 +1,20 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#cmakedefine TLS_KEY_LOG_FILE "@TLS_KEY_LOG_FILE@"

data/ext/build_version.hxx.in

@@ -22,5 +22,5 @@
 #define BACKEND_C_COMPILER "@CMAKE_C_COMPILER_ID@ @CMAKE_C_COMPILER_VERSION@"
 #define BACKEND_SYSTEM "@CMAKE_SYSTEM@"
 #define BACKEND_SYSTEM_PROCESSOR "@CMAKE_SYSTEM_PROCESSOR@"
-#define BACKEND_GIT_REVISION "5b91db73c89347cfb4985490051aea4d82f56c0e"
+#define BACKEND_GIT_REVISION "a23527c044e32c037f4a001c2b2a836716d85d35"
 

data/ext/couchbase/bucket.hxx

@@ -18,21 +18,31 @@
 #pragma once
 
 #include <utility>
+#include <queue>
 
-#include <configuration_monitor.hxx>
 #include <operations.hxx>
+#include <origin.hxx>
 
 namespace couchbase
 {
-
-class bucket
+class bucket : public std::enable_shared_from_this<bucket>
 {
   public:
-    explicit bucket(asio::io_context& ctx, std::string name, configuration config)
-      : ctx_(ctx)
+    explicit bucket(const std::string& client_id,
+                    asio::io_context& ctx,
+                    asio::ssl::context& tls,
+                    std::string name,
+                    couchbase::origin origin,
+                    const std::vector<protocol::hello_feature>& known_features)
+
+      : client_id_(client_id)
+      , ctx_(ctx)
+      , tls_(tls)
       , name_(std::move(name))
-      , config_(std::move(config))
+      , origin_(std::move(origin))
+      , known_features_(known_features)
     {
+        log_prefix_ = fmt::format("[{}/{}]", client_id_, name_);
     }
 
     ~bucket()
@@ -45,31 +55,176 @@ class bucket
         return name_;
     }
 
-    void set_node(size_t index, std::shared_ptr<io::mcbp_session> session)
+    /**
+     * copies nodes from rhs that are not in lhs to output vector
+     */
+    static void diff_nodes(const std::vector<configuration::node>& lhs,
+                           const std::vector<configuration::node>& rhs,
+                           std::vector<configuration::node>& output)
     {
-        if (closed_) {
+        for (const auto& re : rhs) {
+            bool known = false;
+            for (const auto& le : lhs) {
+                if (le.hostname == re.hostname && le.services_plain.management.value_or(0) == re.services_plain.management.value_or(0)) {
+                    known = true;
+                    break;
+                }
+            }
+            if (!known) {
+                output.push_back(re);
+            }
+        }
+    }
+
+    void update_config(const configuration& config)
+    {
+        if (!config_) {
+            spdlog::debug("{} initialize configuration rev={}", log_prefix_, config.rev);
+        } else if (config.rev > config_->rev) {
+            spdlog::debug("{} will update the configuration old={} -> new={}", log_prefix_, config_->rev, config.rev);
+        } else {
             return;
         }
 
-        auto session_manifest = session->manifest();
-        if (!manifest_cache_) {
-            if (session_manifest) {
-                manifest_cache_ = session_manifest.value();
-            }
+        std::vector<configuration::node> added{};
+        std::vector<configuration::node> removed{};
+        if (config_) {
+            diff_nodes(config_->nodes, config.nodes, added);
+            diff_nodes(config.nodes, config_->nodes, removed);
         } else {
-            if (session_manifest && session_manifest->uid > manifest_cache_->uid) {
-                manifest_cache_ = session->manifest().value();
+            added = config.nodes;
+        }
+        config_ = config;
+        if (!added.empty() || removed.empty()) {
+            std::map<size_t, std::shared_ptr<io::mcbp_session>> new_sessions{};
+
+            for (auto entry : sessions_) {
+                std::size_t new_index = config.nodes.size() + 1;
+                for (const auto& node : config.nodes) {
+                    if (entry.second->bootstrap_hostname() == node.hostname_for(origin_.options().network) &&
+                        entry.second->bootstrap_port() ==
+                          std::to_string(node.port_or(origin_.options().network, service_type::kv, origin_.options().enable_tls, 0))) {
+                        new_index = node.index;
+                        break;
+                    }
+                }
+                if (new_index < config.nodes.size()) {
+                    spdlog::debug(R"({} rev={}, preserve session="{}", address="{}:{}")",
+                                  log_prefix_,
+                                  config.rev,
+                                  entry.second->id(),
+                                  entry.second->bootstrap_hostname(),
+                                  entry.second->bootstrap_port());
+                    new_sessions.emplace(new_index, std::move(entry.second));
+                } else {
+                    spdlog::debug(R"({} rev={}, drop session="{}", address="{}:{}")",
+                                  log_prefix_,
+                                  config.rev,
+                                  entry.second->id(),
+                                  entry.second->bootstrap_hostname(),
+                                  entry.second->bootstrap_port());
+                    entry.second.reset();
+                }
+            }
+
+            for (const auto& node : config.nodes) {
+                if (new_sessions.find(node.index) != new_sessions.end()) {
+                    continue;
+                }
+
+                auto hostname = node.hostname_for(origin_.options().network);
+                auto port = node.port_or(origin_.options().network, service_type::kv, origin_.options().enable_tls, 0);
+                couchbase::origin origin(origin_.credentials(), hostname, port, origin_.options());
+                std::shared_ptr<io::mcbp_session> session;
+                if (origin_.options().enable_tls) {
+                    session = std::make_shared<io::mcbp_session>(client_id_, ctx_, tls_, origin, name_, known_features_);
+                } else {
+                    session = std::make_shared<io::mcbp_session>(client_id_, ctx_, origin, name_, known_features_);
+                }
+                spdlog::debug(R"({} rev={}, add session="{}", address="{}:{}")", log_prefix_, config.rev, session->id(), hostname, port);
+                session->bootstrap(
+                  [self = shared_from_this(), session](std::error_code err, const configuration& cfg) {
+                      if (!err) {
+                          self->update_config(cfg);
+                          session->on_configuration_update([self](const configuration& new_config) { self->update_config(new_config); });
+                          session->on_stop([index = session->index(), self](io::retry_reason reason) {
+                              if (reason == io::retry_reason::socket_closed_while_in_flight) {
+                                  self->restart_node(index);
+                              }
+                          });
+                      }
+                  },
+                  true);
+                new_sessions.emplace(node.index, std::move(session));
             }
+            sessions_ = new_sessions;
         }
-        sessions_.emplace(index, std::move(session));
     }
 
-    void remove_node(size_t index)
+    void restart_node(std::size_t index)
     {
-        if (closed_) {
+        auto ptr = sessions_.find(index);
+        if (ptr == sessions_.end()) {
+            spdlog::debug(R"({} requested to restart session idx={}, which does not exist, ignoring)", log_prefix_, index);
             return;
         }
-        sessions_.erase(index);
+        auto& old_session = ptr->second;
+        auto hostname = old_session->bootstrap_hostname();
+        auto port = old_session->bootstrap_port();
+        spdlog::debug(R"({} restarting session idx={}, id="{}", address="{}")", log_prefix_, index, old_session->id(), hostname, port);
+        couchbase::origin origin(origin_.credentials(), hostname, port, origin_.options());
+        sessions_.erase(ptr);
+        std::shared_ptr<io::mcbp_session> session;
+        if (origin_.options().enable_tls) {
+            session = std::make_shared<io::mcbp_session>(client_id_, ctx_, tls_, origin, name_, known_features_);
+        } else {
+            session = std::make_shared<io::mcbp_session>(client_id_, ctx_, origin, name_, known_features_);
+        }
+        session->bootstrap(
+          [self = shared_from_this(), session](std::error_code err, const configuration& config) {
+              if (!err) {
+                  self->update_config(config);
+                  session->on_configuration_update([self](const configuration& new_config) { self->update_config(new_config); });
+              }
+          },
+          true);
+        sessions_.emplace(index, std::move(session));
+    }
+
+    template<typename Handler>
+    void bootstrap(Handler&& handler)
+    {
+        std::shared_ptr<io::mcbp_session> new_session;
+        if (origin_.options().enable_tls) {
+            new_session = std::make_shared<io::mcbp_session>(client_id_, ctx_, tls_, origin_, name_, known_features_);
+        } else {
+            new_session = std::make_shared<io::mcbp_session>(client_id_, ctx_, origin_, name_, known_features_);
+        }
+        new_session->bootstrap([self = shared_from_this(), new_session, h = std::forward<Handler>(handler)](
+                                 std::error_code ec, const configuration& cfg) mutable {
+            if (!ec) {
+                size_t this_index = new_session->index();
+                new_session->on_configuration_update([self](const configuration& config) { self->update_config(config); });
+                new_session->on_stop([this_index, self](io::retry_reason reason) {
+                    if (reason == io::retry_reason::socket_closed_while_in_flight) {
+                        self->restart_node(this_index);
+                    }
+                });
+
+                self->sessions_.emplace(this_index, std::move(new_session));
+                self->update_config(cfg);
+                self->drain_deferred_queue();
+            }
+            h(ec, cfg);
+        });
+    }
+
+    void drain_deferred_queue()
+    {
+        while (!deferred_commands_.empty()) {
+            deferred_commands_.front()();
+            deferred_commands_.pop();
+        }
     }
 
     template<typename Request, typename Handler>
@@ -78,19 +233,16 @@ class bucket
         if (closed_) {
             return;
         }
-        size_t index = 0;
-        std::tie(request.partition, index) = config_.map_key(request.id.key);
-        auto session = sessions_.at(index);
-        if (manifest_cache_) {
-            request.id.collection_uid = get_collection_uid(request.id.collection);
+        auto cmd = std::make_shared<operations::mcbp_command<bucket, Request>>(ctx_, shared_from_this(), request);
+        cmd->start([cmd, handler = std::forward<Handler>(handler)](std::error_code ec, std::optional<io::mcbp_message> msg) mutable {
+            using encoded_response_type = typename Request::encoded_response_type;
+            handler(make_response(ec, cmd->request, msg ? encoded_response_type(*msg) : encoded_response_type{}));
+        });
+        if (config_) {
+            map_and_send(cmd);
         } else {
-            if (!request.id.collection.empty() && request.id.collection != "_default._default") {
-                handler(make_response(std::make_error_code(error::common_errc::unsupported_operation), request, {}));
-                return;
-            }
+            deferred_commands_.emplace([self = shared_from_this(), cmd]() { self->map_and_send(cmd); });
         }
-        auto cmd = std::make_shared<operations::command<Request>>(ctx_, std::move(request));
-        cmd->send_to(session, std::forward<Handler>(handler));
     }
 
     void close()
@@ -99,38 +251,63 @@ class bucket
             return;
         }
         closed_ = true;
+
+        drain_deferred_queue();
         for (auto& session : sessions_) {
-            session.second->stop();
+            session.second->stop(io::retry_reason::do_not_retry);
         }
     }
 
-  private:
-    [[nodiscard]] uint32_t get_collection_uid(const std::string& collection_path)
+    template<typename Request>
+    void map_and_send(std::shared_ptr<operations::mcbp_command<bucket, Request>> cmd)
     {
-        Expects(manifest_cache_.has_value());
-        Expects(!collection_path.empty());
-        auto dot = collection_path.find('.');
-        Expects(dot != std::string::npos);
-        std::string scope = collection_path.substr(0, dot);
-        std::string collection = collection_path.substr(dot + 1);
-        for (const auto& s : manifest_cache_->scopes) {
-            if (s.name == scope) {
-                for (const auto& c : s.collections) {
-                    if (c.name == collection) {
-                        return gsl::narrow_cast<std::uint32_t>(c.uid);
-                    }
-                }
-            }
+        if (closed_) {
+            return cmd->cancel(io::retry_reason::do_not_retry);
+        }
+        std::int16_t index = 0;
+        std::tie(cmd->request.partition, index) = config_->map_key(cmd->request.id.key);
+        if (index < 0) {
+            return io::retry_orchestrator::maybe_retry(
+              cmd->manager_, cmd, io::retry_reason::node_not_available, std::make_error_code(error::common_errc::request_canceled));
         }
-        Ensures(false); // FIXME: return collection not found
-        return 0;
+        cmd->send_to(sessions_.at(static_cast<std::size_t>(index)));
     }
 
+    template<typename Request>
+    void schedule_for_retry(std::shared_ptr<operations::mcbp_command<bucket, Request>> cmd, std::chrono::milliseconds duration)
+    {
+        if (closed_) {
+            return cmd->cancel(io::retry_reason::do_not_retry);
+        }
+        cmd->retry_backoff.expires_after(duration);
+        cmd->retry_backoff.async_wait([self = shared_from_this(), cmd](std::error_code ec) mutable {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            self->map_and_send(cmd);
+        });
+    }
+
+    [[nodiscard]] const std::string& log_prefix()
+    {
+        return log_prefix_;
+    }
+
+  private:
+    std::string client_id_;
     asio::io_context& ctx_;
+    asio::ssl::context& tls_;
     std::string name_;
-    configuration config_;
+    origin origin_;
+
+    std::optional<configuration> config_{};
+    std::vector<protocol::hello_feature> known_features_;
+
+    std::queue<std::function<void()>> deferred_commands_{};
+
     bool closed_{ false };
-    std::optional<collections_manifest> manifest_cache_{};
     std::map<size_t, std::shared_ptr<io::mcbp_session>> sessions_{};
+
+    std::string log_prefix_{};
 };
 } // namespace couchbase