couchbase 3.0.0.beta.1 → 3.0.0

data/examples/search_with_consistency.rb

@@ -1,6 +1,20 @@
-require 'couchbase'
+#    Copyright 2020 Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 
-include Couchbase
+require "couchbase"
+
+include Couchbase # rubocop:disable Style/MixinUsage for brevity
 
 options = Cluster::ClusterOptions.new
 options.authenticate("Administrator", "password")
@@ -18,27 +32,27 @@ rescue Error::IndexNotFound
   index.source_type = "couchbase"
   index.source_name = bucket_name
   index.params = {
-      mapping: {
-          types: {
-              "knob" => {
-                  properties: {
-                      "name" => {
-                          fields: [
-                              {
-                                  name: "name",
-                                  type: "text",
-                                  include_in_all: true,
-                                  include_term_vectors: true,
-                                  index: true,
-                                  store: true,
-                                  docvalues: true,
-                              }
-                          ]
-                      },
-                  }
-              }
-          }
-      }
+    mapping: {
+      types: {
+        "knob" => {
+          properties: {
+            "name" => {
+              fields: [
+                {
+                  name: "name",
+                  type: "text",
+                  include_in_all: true,
+                  include_term_vectors: true,
+                  index: true,
+                  store: true,
+                  docvalues: true,
+                },
+              ],
+            },
+          },
+        },
+      },
+    },
   }
 
   cluster.search_indexes.upsert_index(index)
@@ -48,6 +62,7 @@ rescue Error::IndexNotFound
     sleep(1)
     num = cluster.search_indexes.get_indexed_documents_count(search_index_name)
     break if num_indexed == num
+
     num_indexed = num
     puts "indexing #{search_index_name.inspect}: #{num_indexed} documents"
   end
@@ -59,9 +74,9 @@ collection = cluster.bucket(bucket_name).default_collection
 # and supply mutation tokens from those operations.
 random_string = ("a".."z").to_a.sample(10).join
 res = collection.upsert("user:#{random_string}", {
-    "name" => "Brass Doorknob",
-    "email" => "brass.doorknob@example.com",
-    "data" => random_string,
+  "name" => "Brass Doorknob",
+  "email" => "brass.doorknob@example.com",
+  "data" => random_string,
 })
 
 state = MutationState.new(res.mutation_token)
@@ -74,10 +89,8 @@ options.consistent_with(state)
 res = cluster.search_query(search_index_name, query, options)
 
 res.rows.each do |row|
-  if row.id == "user:#{random_string}"
-    puts "--- Found our newly created document!"
-  end
-  if ENV['REMOVE_DOOR_KNOBS']
+  puts "--- Found our newly created document!" if row.id == "user:#{random_string}"
+  if ENV["REMOVE_DOOR_KNOBS"]
     puts "Removing #{row.id} (requested via env)"
     collection.remove(row.id)
   end

data/examples/subdocument.rb

@@ -1,5 +1,19 @@
-require 'couchbase'
-include Couchbase
+#    Copyright 2020 Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require "couchbase"
+include Couchbase # rubocop:disable Style/MixinUsage for brevity
 
 options = Cluster::ClusterOptions.new
 options.authenticate("Administrator", "password")
@@ -8,44 +22,42 @@ bucket = cluster.bucket("default")
 collection = bucket.default_collection
 
 document = {
-    name: "Douglas Reynholm",
-    email: "douglas@reynholmindustries.com",
-    addresses: {
-        billing: {
-            line1: "123 Any Street",
-            line2: "Anytown",
-            country: "United Kingdom"
-        },
-        delivery: {
-            line1: "123 Any Street",
-            line2: "Anytown",
-            country: "United Kingdom"
-        }
+  name: "Douglas Reynholm",
+  email: "douglas@reynholmindustries.com",
+  addresses: {
+    billing: {
+      line1: "123 Any Street",
+      line2: "Anytown",
+      country: "United Kingdom",
+    },
+    delivery: {
+      line1: "123 Any Street",
+      line2: "Anytown",
+      country: "United Kingdom",
     },
-    purchases: {
-        complete: [339, 976, 442, 666],
-        abandoned: [157, 42, 999]
-    }
+  },
+  purchases: {
+    complete: [339, 976, 442, 666],
+    abandoned: [157, 42, 999],
+  },
 }
 collection.upsert("customer123", document)
 
 res = collection.mutate_in("customer123", [
-    MutateInSpec.upsert("fax", "311-555-0151")
-])
+                             MutateInSpec.upsert("fax", "311-555-0151"),
+                           ])
 puts "The document has been modified successfully: cas=#{res.cas}"
 
 res = collection.mutate_in("customer123", [
-    MutateInSpec.upsert("_framework.model_type", "Customer").xattr,
-    MutateInSpec.remove("addresses.billing[2]"),
-    MutateInSpec.replace("email", "dougr96@hotmail.com"),
-])
+                             MutateInSpec.upsert("_framework.model_type", "Customer").xattr,
+                             MutateInSpec.remove("addresses.billing[2]"),
+                             MutateInSpec.replace("email", "dougr96@hotmail.com"),
+                           ])
 puts "The document has been modified successfully: cas=#{res.cas}"
 
 res = collection.lookup_in "customer123", [
-    LookupInSpec.get("addresses.delivery.country"),
-    LookupInSpec.exists("purchases.pending[-1]"),
+  LookupInSpec.get("addresses.delivery.country"),
+  LookupInSpec.exists("purchases.pending[-1]"),
 ]
 puts "The customer's delivery country is #{res.content(0)}"
-if res.exists?(1)
-  puts "The customer has pending purchases"
-end
+puts "The customer has pending purchases" if res.exists?(1)

data/examples/view.rb

@@ -1,8 +1,20 @@
-# coding: utf-8
+#    Copyright 2020 Couchbase, Inc.
 #
-require 'couchbase'
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require "couchbase"
 
-include Couchbase
+include Couchbase # rubocop:disable Style/MixinUsage for brevity
 
 options = Cluster::ClusterOptions.new
 options.authenticate("Administrator", "password")
@@ -26,17 +38,16 @@ options.order = :descending
 res = bucket.view_query("beer", "brewery_beers", options)
 puts "\nTotal documents in 'beer/brewery_beers' index: #{res.meta_data.total_rows}"
 puts "Last #{options.limit} documents:"
-res.rows.each_with_index do |row|
+res.rows.each do |row|
   doc = collection.get(row.id)
   puts "#{row.id} (type: #{doc.content['type']}, key: #{row.key})"
 end
 
-
 random_number = rand(0..1_000_000_000)
 unique_brewery_id = "random_brewery:#{random_number}"
 collection.upsert("random_brewery:#{random_number}", {
-    "name" => "Random brewery: #{random_number}",
-    "type" => "brewery"
+  "name" => "Random brewery: #{random_number}",
+  "type" => "brewery",
 })
 puts "\nRequest with consistency. Generated brewery name: #{unique_brewery_id}"
 options = Bucket::ViewOptions.new
@@ -44,7 +55,5 @@ options.start_key = ["random_brewery:"]
 options.scan_consistency = :request_plus
 res = bucket.view_query("beer", "brewery_beers", options)
 res.rows.each do |row|
-  if row.id == unique_brewery_id
-    puts "Found newly created document: #{collection.get(row.id).content}"
-  end
+  puts "Found newly created document: #{collection.get(row.id).content}" if row.id == unique_brewery_id
 end

data/ext/CMakeLists.txt

@@ -8,6 +8,36 @@ if(CCACHE)
     set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE})
 endif()
 
+include(CMakePushCheckState)
+include(CheckSymbolExists)
+
+cmake_push_check_state(RESET)
+find_library(EXECINFO_LIBRARY NAMES execinfo)
+if(EXECINFO_LIBRARY)
+    set(CMAKE_REQUIRED_LIBRARIES "${EXECINFO_LIBRARY}")
+    list(APPEND PLATFORM_LIBRARIES "${EXECINFO_LIBRARY}")
+endif(EXECINFO_LIBRARY)
+check_symbol_exists(backtrace execinfo.h HAVE_BACKTRACE)
+cmake_pop_check_state()
+
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_DEFINITIONS "-D_GNU_SOURCE")
+find_library(DL_LIBRARY NAMES dl)
+if(DL_LIBRARY)
+    set(CMAKE_REQUIRED_LIBRARIES "${DL_LIBRARY}")
+    list(APPEND PLATFORM_LIBRARIES "${DL_LIBRARY}")
+endif(DL_LIBRARY)
+check_symbol_exists(dladdr dlfcn.h HAVE_DLADDR)
+cmake_pop_check_state()
+
+if(HAVE_BACKTRACE)
+    add_definitions(-DHAVE_BACKTRACE=1)
+endif()
+
+if(HAVE_DLADDR)
+    add_definitions(-DHAVE_DLADDR=1)
+endif()
+
 add_subdirectory(third_party/gsl)
 add_subdirectory(third_party/json)
 add_subdirectory(third_party/spdlog)
@@ -90,6 +120,12 @@ else()
                       -Wuseless-cast # warn if you perform a cast to the same type
         )
     endif()
+
+    if(HAVE_BACKTRACE OR HAVE_DLADDR)
+        target_compile_options(project_options INTERFACE -ggdb3)
+        target_link_libraries(project_options INTERFACE -rdynamic)
+        add_definitions(-D_GNU_SOURCE=1)
+    endif()
 endif()
 
 # Read more at https://wiki.wireshark.org/TLS
@@ -102,8 +138,10 @@ message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")
 
 include_directories(${CMAKE_SOURCE_DIR}/couchbase)
 
-add_library(platform OBJECT couchbase/platform/string_hex.cc couchbase/platform/uuid.cc couchbase/platform/random.cc
-                            couchbase/platform/base64.cc)
+add_library(
+    platform OBJECT couchbase/platform/string_hex.cc couchbase/platform/uuid.cc couchbase/platform/random.cc
+                    couchbase/platform/base64.cc couchbase/platform/backtrace.c couchbase/platform/terminate_handler.cc)
+target_include_directories(platform PRIVATE ${PROJECT_BINARY_DIR}/generated)
 
 add_library(cbcrypto OBJECT couchbase/cbcrypto/cbcrypto.cc)
 

data/ext/build_version.hxx.in

@@ -22,5 +22,5 @@
 #define BACKEND_C_COMPILER "@CMAKE_C_COMPILER_ID@ @CMAKE_C_COMPILER_VERSION@"
 #define BACKEND_SYSTEM "@CMAKE_SYSTEM@"
 #define BACKEND_SYSTEM_PROCESSOR "@CMAKE_SYSTEM_PROCESSOR@"
-#define BACKEND_GIT_REVISION "f01d0c237750cc67ee75325ed160bc0da1d2bf7a"
+#define BACKEND_GIT_REVISION "a23527c044e32c037f4a001c2b2a836716d85d35"
 

data/ext/couchbase/bucket.hxx

@@ -42,6 +42,7 @@ class bucket : public std::enable_shared_from_this<bucket>
       , origin_(std::move(origin))
       , known_features_(known_features)
     {
+        log_prefix_ = fmt::format("[{}/{}]", client_id_, name_);
     }
 
     ~bucket()
@@ -54,6 +55,142 @@ class bucket : public std::enable_shared_from_this<bucket>
         return name_;
     }
 
+    /**
+     * copies nodes from rhs that are not in lhs to output vector
+     */
+    static void diff_nodes(const std::vector<configuration::node>& lhs,
+                           const std::vector<configuration::node>& rhs,
+                           std::vector<configuration::node>& output)
+    {
+        for (const auto& re : rhs) {
+            bool known = false;
+            for (const auto& le : lhs) {
+                if (le.hostname == re.hostname && le.services_plain.management.value_or(0) == re.services_plain.management.value_or(0)) {
+                    known = true;
+                    break;
+                }
+            }
+            if (!known) {
+                output.push_back(re);
+            }
+        }
+    }
+
+    void update_config(const configuration& config)
+    {
+        if (!config_) {
+            spdlog::debug("{} initialize configuration rev={}", log_prefix_, config.rev);
+        } else if (config.rev > config_->rev) {
+            spdlog::debug("{} will update the configuration old={} -> new={}", log_prefix_, config_->rev, config.rev);
+        } else {
+            return;
+        }
+
+        std::vector<configuration::node> added{};
+        std::vector<configuration::node> removed{};
+        if (config_) {
+            diff_nodes(config_->nodes, config.nodes, added);
+            diff_nodes(config.nodes, config_->nodes, removed);
+        } else {
+            added = config.nodes;
+        }
+        config_ = config;
+        if (!added.empty() || removed.empty()) {
+            std::map<size_t, std::shared_ptr<io::mcbp_session>> new_sessions{};
+
+            for (auto entry : sessions_) {
+                std::size_t new_index = config.nodes.size() + 1;
+                for (const auto& node : config.nodes) {
+                    if (entry.second->bootstrap_hostname() == node.hostname_for(origin_.options().network) &&
+                        entry.second->bootstrap_port() ==
+                          std::to_string(node.port_or(origin_.options().network, service_type::kv, origin_.options().enable_tls, 0))) {
+                        new_index = node.index;
+                        break;
+                    }
+                }
+                if (new_index < config.nodes.size()) {
+                    spdlog::debug(R"({} rev={}, preserve session="{}", address="{}:{}")",
+                                  log_prefix_,
+                                  config.rev,
+                                  entry.second->id(),
+                                  entry.second->bootstrap_hostname(),
+                                  entry.second->bootstrap_port());
+                    new_sessions.emplace(new_index, std::move(entry.second));
+                } else {
+                    spdlog::debug(R"({} rev={}, drop session="{}", address="{}:{}")",
+                                  log_prefix_,
+                                  config.rev,
+                                  entry.second->id(),
+                                  entry.second->bootstrap_hostname(),
+                                  entry.second->bootstrap_port());
+                    entry.second.reset();
+                }
+            }
+
+            for (const auto& node : config.nodes) {
+                if (new_sessions.find(node.index) != new_sessions.end()) {
+                    continue;
+                }
+
+                auto hostname = node.hostname_for(origin_.options().network);
+                auto port = node.port_or(origin_.options().network, service_type::kv, origin_.options().enable_tls, 0);
+                couchbase::origin origin(origin_.credentials(), hostname, port, origin_.options());
+                std::shared_ptr<io::mcbp_session> session;
+                if (origin_.options().enable_tls) {
+                    session = std::make_shared<io::mcbp_session>(client_id_, ctx_, tls_, origin, name_, known_features_);
+                } else {
+                    session = std::make_shared<io::mcbp_session>(client_id_, ctx_, origin, name_, known_features_);
+                }
+                spdlog::debug(R"({} rev={}, add session="{}", address="{}:{}")", log_prefix_, config.rev, session->id(), hostname, port);
+                session->bootstrap(
+                  [self = shared_from_this(), session](std::error_code err, const configuration& cfg) {
+                      if (!err) {
+                          self->update_config(cfg);
+                          session->on_configuration_update([self](const configuration& new_config) { self->update_config(new_config); });
+                          session->on_stop([index = session->index(), self](io::retry_reason reason) {
+                              if (reason == io::retry_reason::socket_closed_while_in_flight) {
+                                  self->restart_node(index);
+                              }
+                          });
+                      }
+                  },
+                  true);
+                new_sessions.emplace(node.index, std::move(session));
+            }
+            sessions_ = new_sessions;
+        }
+    }
+
+    void restart_node(std::size_t index)
+    {
+        auto ptr = sessions_.find(index);
+        if (ptr == sessions_.end()) {
+            spdlog::debug(R"({} requested to restart session idx={}, which does not exist, ignoring)", log_prefix_, index);
+            return;
+        }
+        auto& old_session = ptr->second;
+        auto hostname = old_session->bootstrap_hostname();
+        auto port = old_session->bootstrap_port();
+        spdlog::debug(R"({} restarting session idx={}, id="{}", address="{}")", log_prefix_, index, old_session->id(), hostname, port);
+        couchbase::origin origin(origin_.credentials(), hostname, port, origin_.options());
+        sessions_.erase(ptr);
+        std::shared_ptr<io::mcbp_session> session;
+        if (origin_.options().enable_tls) {
+            session = std::make_shared<io::mcbp_session>(client_id_, ctx_, tls_, origin, name_, known_features_);
+        } else {
+            session = std::make_shared<io::mcbp_session>(client_id_, ctx_, origin, name_, known_features_);
+        }
+        session->bootstrap(
+          [self = shared_from_this(), session](std::error_code err, const configuration& config) {
+              if (!err) {
+                  self->update_config(config);
+                  session->on_configuration_update([self](const configuration& new_config) { self->update_config(new_config); });
+              }
+          },
+          true);
+        sessions_.emplace(index, std::move(session));
+    }
+
     template<typename Handler>
     void bootstrap(Handler&& handler)
     {
@@ -66,52 +203,37 @@ class bucket : public std::enable_shared_from_this<bucket>
         new_session->bootstrap([self = shared_from_this(), new_session, h = std::forward<Handler>(handler)](
                                  std::error_code ec, const configuration& cfg) mutable {
             if (!ec) {
-                self->config_ = cfg;
                 size_t this_index = new_session->index();
-                self->sessions_.emplace(this_index, std::move(new_session));
-                if (cfg.nodes.size() > 1) {
-                    for (const auto& n : cfg.nodes) {
-                        if (n.index != this_index) {
-                            couchbase::origin origin(
-                              self->origin_.get_username(),
-                              self->origin_.get_password(),
-                              n.hostname_for(self->origin_.options().network),
-                              n.port_or(self->origin_.options().network, service_type::kv, self->origin_.options().enable_tls, 0),
-                              self->origin_.options());
-                            std::shared_ptr<io::mcbp_session> s;
-                            if (self->origin_.options().enable_tls) {
-                                s = std::make_shared<io::mcbp_session>(
-                                  self->client_id_, self->ctx_, self->tls_, origin, self->name_, self->known_features_);
-                            } else {
-                                s = std::make_shared<io::mcbp_session>(
-                                  self->client_id_, self->ctx_, origin, self->name_, self->known_features_);
-                            }
-                            s->bootstrap([host = n.hostname, bucket = self->name_](std::error_code err, const configuration& /*config*/) {
-                                // TODO: retry, we know that auth is correct
-                                if (err) {
-                                    spdlog::warn("unable to bootstrap node {} ({}): {}", host, bucket, err.message());
-                                }
-                            });
-                            self->sessions_.emplace(n.index, std::move(s));
-                        }
+                new_session->on_configuration_update([self](const configuration& config) { self->update_config(config); });
+                new_session->on_stop([this_index, self](io::retry_reason reason) {
+                    if (reason == io::retry_reason::socket_closed_while_in_flight) {
+                        self->restart_node(this_index);
                     }
-                }
-                while (!self->deferred_commands_.empty()) {
-                    self->deferred_commands_.front()();
-                    self->deferred_commands_.pop();
-                }
+                });
+
+                self->sessions_.emplace(this_index, std::move(new_session));
+                self->update_config(cfg);
+                self->drain_deferred_queue();
             }
             h(ec, cfg);
         });
     }
 
+    void drain_deferred_queue()
+    {
+        while (!deferred_commands_.empty()) {
+            deferred_commands_.front()();
+            deferred_commands_.pop();
+        }
+    }
+
     template<typename Request, typename Handler>
     void execute(Request request, Handler&& handler)
     {
         if (closed_) {
             return;
         }
-        auto cmd = std::make_shared<operations::mcbp_command<Request>>(ctx_, request);
+        auto cmd = std::make_shared<operations::mcbp_command<bucket, Request>>(ctx_, shared_from_this(), request);
         cmd->start([cmd, handler = std::forward<Handler>(handler)](std::error_code ec, std::optional<io::mcbp_message> msg) mutable {
             using encoded_response_type = typename Request::encoded_response_type;
             handler(make_response(ec, cmd->request, msg ? encoded_response_type(*msg) : encoded_response_type{}));
@@ -129,18 +251,46 @@ class bucket : public std::enable_shared_from_this<bucket>
             return;
         }
         closed_ = true;
+
+        drain_deferred_queue();
         for (auto& session : sessions_) {
-            session.second->stop();
+            session.second->stop(io::retry_reason::do_not_retry);
         }
     }
 
     template<typename Request>
-    void map_and_send(std::shared_ptr<operations::mcbp_command<Request>> cmd)
+    void map_and_send(std::shared_ptr<operations::mcbp_command<bucket, Request>> cmd)
     {
-        size_t index = 0;
+        if (closed_) {
+            return cmd->cancel(io::retry_reason::do_not_retry);
+        }
+        std::int16_t index = 0;
         std::tie(cmd->request.partition, index) = config_->map_key(cmd->request.id.key);
-        auto session = sessions_.at(index);
-        cmd->send_to(session);
+        if (index < 0) {
+            return io::retry_orchestrator::maybe_retry(
+              cmd->manager_, cmd, io::retry_reason::node_not_available, std::make_error_code(error::common_errc::request_canceled));
+        }
+        cmd->send_to(sessions_.at(static_cast<std::size_t>(index)));
+    }
+
+    template<typename Request>
+    void schedule_for_retry(std::shared_ptr<operations::mcbp_command<bucket, Request>> cmd, std::chrono::milliseconds duration)
+    {
+        if (closed_) {
+            return cmd->cancel(io::retry_reason::do_not_retry);
+        }
+        cmd->retry_backoff.expires_after(duration);
+        cmd->retry_backoff.async_wait([self = shared_from_this(), cmd](std::error_code ec) mutable {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            self->map_and_send(cmd);
+        });
+    }
+
+    [[nodiscard]] const std::string& log_prefix()
+    {
+        return log_prefix_;
     }
 
   private:
@@ -157,5 +307,7 @@ class bucket : public std::enable_shared_from_this<bucket>
 
     bool closed_{ false };
     std::map<size_t, std::shared_ptr<io::mcbp_session>> sessions_{};
+
+    std::string log_prefix_{};
 };
 } // namespace couchbase