couchbase 3.4.3 → 3.4.5

data/ext/couchbase/core/logger/logger.cxx

@@ -22,7 +22,8 @@
 #include <spdlog/sinks/stdout_color_sinks.h>
 #include <spdlog/spdlog.h>
 
-static const std::string logger_name{ "couchbase_cxx_client_file_logger" };
+static const std::string file_logger_name{ "couchbase_cxx_client_file_logger" };
+static const std::string protocol_logger_name{ "couchbase_cxx_client_protocol_logger" };
 
 /**
  * Custom log pattern which the loggers will use.
@@ -39,7 +40,12 @@ static const std::string log_pattern{ "[%Y-%m-%d %T.%e] [%P,%t] [%^%l%$] %oms, %
  * messages and send them to the sinks, which do the actual writing (to file,
  * to stream etc.) or further processing.
  */
-static std::shared_ptr<spdlog::logger> file_logger;
+static std::shared_ptr<spdlog::logger> file_logger{};
+
+/**
+ * Instance of the protocol logger.
+ */
+static std::shared_ptr<spdlog::logger> protocol_logger{};
 
 namespace couchbase::core::logger
 {
@@ -146,13 +152,11 @@ is_initialized()
     return file_logger != nullptr;
 }
 
-/**
- * Initialises the loggers. Called if the logger configuration is
- * specified in a separate settings object.
- */
-std::optional<std::string>
-create_file_logger(const configuration& logger_settings)
+std::pair<std::optional<std::string>, std::shared_ptr<spdlog::logger>>
+create_file_logger_impl(const std::string logger_name, const configuration& logger_settings)
 {
+    std::shared_ptr<spdlog::logger> logger{};
+
     auto fname = logger_settings.filename;
     auto buffersz = logger_settings.buffer_size;
     auto cyclesz = logger_settings.cycle_size;
@@ -218,7 +222,7 @@ create_file_logger(const configuration& logger_settings)
         spdlog::drop(logger_name);
 
         if (logger_settings.unit_test) {
-            file_logger = std::make_shared<spdlog::logger>(logger_name, sink);
+            logger = std::make_shared<spdlog::logger>(logger_name, sink);
         } else {
             // Create the default thread pool for async logging
             spdlog::init_thread_pool(buffersz, 1);
@@ -226,23 +230,71 @@ create_file_logger(const configuration& logger_settings)
             // Get the thread pool so that we can actually construct the
             // object with already created sinks...
             auto tp = spdlog::thread_pool();
-            file_logger = std::make_shared<spdlog::async_logger>(logger_name, sink, tp, spdlog::async_overflow_policy::block);
+            logger = std::make_shared<spdlog::async_logger>(logger_name, sink, tp, spdlog::async_overflow_policy::block);
         }
 
-        file_logger->set_pattern(log_pattern);
-        file_logger->set_level(translate_level(logger_settings.log_level));
+        logger->set_pattern(log_pattern);
+        logger->set_level(translate_level(logger_settings.log_level));
 
         // Set the flushing interval policy
         spdlog::flush_every(std::chrono::seconds(1));
 
-        spdlog::register_logger(file_logger);
+        spdlog::register_logger(logger);
     } catch (const spdlog::spdlog_ex& ex) {
         std::string msg = std::string{ "Log initialization failed: " } + ex.what();
-        return std::optional<std::string>{ msg };
+        return { msg, {} };
+    }
+    return { {}, logger };
+}
+
+/**
+ * Initialises the loggers. Called if the logger configuration is
+ * specified in a separate settings object.
+ */
+std::optional<std::string>
+create_file_logger(const configuration& logger_settings)
+{
+    auto [error, logger] = create_file_logger_impl(file_logger_name, logger_settings);
+    if (error) {
+        return error;
+    }
+    file_logger = std::move(logger);
+    return {};
+}
+
+std::optional<std::string>
+create_protocol_logger(const configuration& logger_settings)
+{
+    if (logger_settings.filename.empty()) {
+        return "File name is missing";
+    }
+    auto config = logger_settings;
+    config.log_level = couchbase::core::logger::level::trace;
+    auto [error, logger] = create_file_logger_impl(protocol_logger_name, config);
+    if (error) {
+        return error;
     }
+    protocol_logger = std::move(logger);
     return {};
 }
 
+bool
+should_log_protocol()
+{
+    return protocol_logger != nullptr;
+}
+
+namespace detail
+{
+void
+log_protocol(const char* file, int line, const char* function, std::string_view msg)
+{
+    if (should_log_protocol()) {
+        return protocol_logger->log(spdlog::source_loc{ file, line, function }, spdlog::level::level_enum::trace, msg);
+    }
+}
+} // namespace detail
+
 spdlog::logger*
 get()
 {
@@ -252,17 +304,20 @@ get()
 void
 reset()
 {
-    spdlog::drop(logger_name);
+    spdlog::drop(file_logger_name);
     file_logger.reset();
+
+    spdlog::drop(protocol_logger_name);
+    protocol_logger.reset();
 }
 
 void
 create_blackhole_logger()
 {
     // delete if already exists
-    spdlog::drop(logger_name);
+    spdlog::drop(file_logger_name);
 
-    file_logger = std::make_shared<spdlog::logger>(logger_name, std::make_shared<spdlog::sinks::null_sink_mt>());
+    file_logger = std::make_shared<spdlog::logger>(file_logger_name, std::make_shared<spdlog::sinks::null_sink_mt>());
 
     file_logger->set_level(spdlog::level::off);
     file_logger->set_pattern(log_pattern);
@@ -274,11 +329,11 @@ void
 create_console_logger()
 {
     // delete if already exists
-    spdlog::drop(logger_name);
+    spdlog::drop(file_logger_name);
 
     auto stderrsink = std::make_shared<spdlog::sinks::stderr_color_sink_mt>();
 
-    file_logger = std::make_shared<spdlog::logger>(logger_name, stderrsink);
+    file_logger = std::make_shared<spdlog::logger>(file_logger_name, stderrsink);
     file_logger->set_level(spdlog::level::info);
     file_logger->set_pattern(log_pattern);
 
@@ -322,8 +377,13 @@ void
 set_log_levels(level lvl)
 {
     auto level = translate_level(lvl);
-    // Apply the function to each registered spdlog::logger
+    // Apply the function to each registered spdlog::logger except protocol logger
     spdlog::apply_all([level](std::shared_ptr<spdlog::logger> l) {
+        if (l->name() == protocol_logger_name) {
+            l->set_level(spdlog::level::trace);
+            return;
+        }
+
         try {
             l->set_level(level);
         } catch (const spdlog::spdlog_ex& e) {

data/ext/couchbase/core/logger/logger.hxx

@@ -54,6 +54,17 @@ level_from_str(const std::string& str);
 std::optional<std::string>
 create_file_logger(const configuration& logger_settings);
 
+/**
+ * Protocol logger writes only communication logs with the nodes.
+ *
+ * It accepts the same settings as a file logger, and writes only to the filesystem.
+ *
+ * @param logger_settings
+ * @return
+ */
+std::optional<std::string>
+create_protocol_logger(const configuration& logger_settings);
+
 /**
  * Initialize the logger with the blackhole logger object
  *
@@ -144,6 +155,9 @@ set_log_levels(level lvl);
 bool
 should_log(level lvl);
 
+bool
+should_log_protocol();
+
 namespace detail
 {
 /**
@@ -153,6 +167,9 @@ namespace detail
  */
 void
 log(const char* file, int line, const char* function, level lvl, std::string_view msg);
+
+void
+log_protocol(const char* file, int line, const char* function, std::string_view msg);
 } // namespace detail
 
 /**
@@ -168,6 +185,13 @@ log(const char* file, int line, const char* function, level lvl, const String& m
     detail::log(file, line, function, lvl, fmt::format(msg, std::forward<Args>(args)...));
 }
 
+template<typename String, typename... Args>
+inline void
+log_protocol(const char* file, int line, const char* function, const String& msg, Args&&... args)
+{
+    detail::log_protocol(file, line, function, fmt::format(msg, std::forward<Args>(args)...));
+}
+
 /**
  * Tell the logger to flush its buffers
  */
@@ -214,6 +238,13 @@ is_initialized();
 #define CB_LOG_CRITICAL(...)                                                                                                               \
     COUCHBASE_LOG(__FILE__, __LINE__, COUCHBASE_LOGGER_FUNCTION, couchbase::core::logger::level::critical, __VA_ARGS__)
 
+#define CB_LOG_PROTOCOL(...)                                                                                                               \
+    do {                                                                                                                                   \
+        if (couchbase::core::logger::should_log_protocol()) {                                                                              \
+            couchbase::core::logger::log_protocol(__FILE__, __LINE__, COUCHBASE_LOGGER_FUNCTION, __VA_ARGS__);                             \
+        }                                                                                                                                  \
+    } while (false)
+
 /**
  * Convenience macros which log with the given level, and message, if the given
  * level is currently enabled.

data/ext/couchbase/core/management/bucket_settings.hxx

@@ -105,16 +105,19 @@ struct bucket_settings {
 
     std::string name;
     std::string uuid;
+    std::uint64_t ram_quota_mb{ 0 }; // If not explicitly set, defaults to 100 on create_bucket, unset on update_bucket
     cluster::bucket_type bucket_type{ cluster::bucket_type::unknown };
-    std::uint64_t ram_quota_mb{ 100 };
-    std::uint32_t max_expiry{ 0 };
+    std::optional<std::uint32_t> max_expiry{};
     bucket_compression compression_mode{ bucket_compression::unknown };
     std::optional<couchbase::durability_level> minimum_durability_level{};
-    std::uint32_t num_replicas{ 1 };
-    bool replica_indexes{ false };
-    bool flush_enabled{ false };
+    std::optional<std::uint32_t> num_replicas{};
+    std::optional<bool> replica_indexes{};
+    std::optional<bool> flush_enabled{};
     bucket_eviction_policy eviction_policy{ bucket_eviction_policy::unknown };
     bucket_conflict_resolution conflict_resolution_type{ bucket_conflict_resolution::unknown };
+    std::optional<bool> history_retention_collection_default{};
+    std::optional<std::uint32_t> history_retention_bytes;
+    std::optional<std::uint32_t> history_retention_duration{};
 
     /**
      * UNCOMMITTED: This API may change in the future

data/ext/couchbase/core/management/bucket_settings_json.hxx

@@ -33,10 +33,20 @@ struct traits<couchbase::core::management::cluster::bucket_settings> {
         result.uuid = v.at("uuid").get_string();
         const static std::uint64_t megabyte = 1024LLU * 1024LLU;
         result.ram_quota_mb = v.at("quota").at("rawRAM").get_unsigned() / megabyte;
-        result.num_replicas = v.at("replicaNumber").template as<std::uint32_t>();
+        result.num_replicas = v.at("replicaNumber").template as<std::optional<std::uint32_t>>();
 
         if (auto* max_ttl = v.find("maxTTL"); max_ttl != nullptr) {
-            result.max_expiry = max_ttl->template as<std::uint32_t>();
+            result.max_expiry = max_ttl->template as<std::optional<std::uint32_t>>();
+        }
+
+        if (auto* history_retention_default = v.find("historyRetentionCollectionDefault"); history_retention_default != nullptr) {
+            result.history_retention_collection_default = history_retention_default->template as<std::optional<bool>>();
+        }
+        if (auto* history_retention_bytes = v.find("historyRetentionBytes"); history_retention_bytes != nullptr) {
+            result.history_retention_bytes = history_retention_bytes->template as<std::optional<std::uint32_t>>();
+        }
+        if (auto* history_retention_duration = v.find("historyRetentionSeconds"); history_retention_duration != nullptr) {
+            result.history_retention_duration = history_retention_duration->template as<std::optional<std::uint32_t>>();
         }
 
         if (auto& str = v.at("bucketType").get_string(); str == "couchbase" || str == "membase") {

data/ext/couchbase/core/meta/features.hxx

@@ -29,3 +29,45 @@
  * couchbase::core::meta::sdk_version() function is available
  */
 #define COUCHBASE_CXX_CLIENT_HAS_SDK_SEMVER 1
+
+/**
+ * couchbase::core::cluster_options and couchbase::security_options support
+ * passing TLS trust certificate by value
+ */
+#define COUCHBASE_CXX_CLIENT_CAN_PASS_TLS_TRUST_CERTIFICATE_BY_VALUE 1
+
+/**
+ *  Range scan is available in the core
+ *  couchbase::core::range_scan_orchestrator and relevant options in the core API
+ */
+#define COUCHBASE_CXX_CLIENT_CORE_HAS_RANGE_SCAN 1
+
+/**
+ * Query with reads from replica is available:
+ *  - use_replica field in couchbase::core::operations::query_request
+ *  - couchbase::query_options::use_replica()
+ */
+#define COUCHBASE_CXX_CLIENT_QUERY_READ_FROM_REPLICA 1
+
+/**
+ * Subdoc read from replica is available in the core
+ * couchbase::core::lookup_in_replica support
+ */
+#define COUCHBASE_CXX_CLIENT_CORE_HAS_SUBDOC_READ_REPLICA 1
+
+/**
+ * Collection management is accessible from the public API
+ * couchbase::bucket::collections() support
+ */
+#define COUCHBASE_CXX_CLIENT_HAS_PUBLIC_COLLECTION_MANAGEMENT 1
+
+/**
+ * Support for bucket no-deduplication feature in bucket and collection management
+ */
+#define COUCHBASE_CXX_CLIENT_HAS_BUCKET_NO_DEDUP 1
+
+/**
+ * Collection query index management is available in the public API
+ * couchbase::collection::query_indexes() support
+ */
+#define COUCHBASE_CXX_CLIENT_HAS_COLLECTION_QUERY_INDEX_MANAGEMENT 1

data/ext/couchbase/core/operations/document_lookup_in.cxx

@@ -70,6 +70,9 @@ lookup_in_request::make_response(key_value_error_context&& ctx, const encoded_re
             fields[i].status = res_entry.status;
             fields[i].ec =
               protocol::map_status_code(protocol::client_opcode::subdoc_multi_mutation, static_cast<std::uint16_t>(res_entry.status));
+            if (fields[i].opcode == protocol::subdoc_opcode::exists && fields[i].ec == errc::key_value::path_not_found) {
+                fields[i].ec.clear();
+            }
             if (!fields[i].ec && !ctx.ec()) {
                 ec = fields[i].ec;
             }
@@ -79,7 +82,11 @@ lookup_in_request::make_response(key_value_error_context&& ctx, const encoded_re
             }
             fields[i].exists =
               res_entry.status == key_value_status_code::success || res_entry.status == key_value_status_code::subdoc_success_deleted;
-            fields[i].value = utils::to_binary(res_entry.value);
+            if (fields[i].opcode == protocol::subdoc_opcode::exists && !fields[i].ec) {
+                fields[i].value = utils::json::generate_binary(fields[i].exists);
+            } else {
+                fields[i].value = utils::to_binary(res_entry.value);
+            }
         }
         if (!ec) {
             cas = encoded.cas();

data/ext/couchbase/core/operations/document_lookup_in_all_replicas.hxx

@@ -0,0 +1,192 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2020-2021 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include "core/error_context/key_value.hxx"
+#include "core/impl/lookup_in_replica.hxx"
+#include "core/impl/subdoc/command.hxx"
+#include "core/operations/document_lookup_in.hxx"
+#include "core/operations/operation_traits.hxx"
+#include "core/utils/movable_function.hxx"
+#include "couchbase/codec/encoded_value.hxx"
+#include "couchbase/error_codes.hxx"
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+namespace couchbase::core::operations
+{
+struct lookup_in_all_replicas_response {
+    struct entry {
+        struct lookup_in_entry {
+            std::string path;
+            codec::binary value;
+            std::size_t original_index;
+            bool exists;
+            protocol::subdoc_opcode opcode;
+            key_value_status_code status;
+            std::error_code ec{};
+        };
+        std::vector<lookup_in_entry> fields{};
+        couchbase::cas cas{};
+        bool deleted{ false };
+        bool is_replica{ true };
+    };
+    subdocument_error_context ctx{};
+    std::vector<entry> entries{};
+};
+
+struct lookup_in_all_replicas_request {
+    using response_type = lookup_in_all_replicas_response;
+    using encoded_request_type = core::protocol::client_request<core::protocol::lookup_in_replica_request_body>;
+    using encoded_response_type = core::protocol::client_response<core::protocol::lookup_in_replica_response_body>;
+
+    core::document_id id;
+    std::vector<couchbase::core::impl::subdoc::command> specs{};
+    std::optional<std::chrono::milliseconds> timeout{};
+    std::shared_ptr<couchbase::tracing::request_span> parent_span{ nullptr };
+
+    template<typename Core, typename Handler>
+    void execute(Core core, Handler handler)
+    {
+        core->with_bucket_configuration(
+          id.bucket(),
+          [core, id = id, timeout = timeout, specs = specs, parent_span = parent_span, h = std::forward<Handler>(handler)](
+            std::error_code ec, const topology::configuration& config) mutable {
+              if (!config.supports_subdoc_read_replica()) {
+                  ec = errc::common::feature_not_available;
+              }
+
+              if (ec) {
+                  std::optional<std::string> first_error_path{};
+                  std::optional<std::size_t> first_error_index{};
+                  return h(response_type{
+                    make_subdocument_error_context(make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) });
+              }
+              using handler_type = utils::movable_function<void(response_type)>;
+
+              struct replica_context {
+                  replica_context(handler_type handler, std::uint32_t expected_responses)
+                    : handler_(std::move(handler))
+                    , expected_responses_(expected_responses)
+                  {
+                  }
+
+                  handler_type handler_;
+                  std::uint32_t expected_responses_;
+                  bool done_{ false };
+                  std::mutex mutex_{};
+                  std::vector<lookup_in_all_replicas_response::entry> result_{};
+              };
+              auto ctx = std::make_shared<replica_context>(std::move(h), config.num_replicas.value_or(0U) + 1U);
+
+              for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) {
+                  document_id replica_id{ id };
+                  replica_id.node_index(idx);
+                  core->execute(impl::lookup_in_replica_request{ std::move(replica_id), specs, timeout, parent_span },
+                                [ctx](impl::lookup_in_replica_response&& resp) {
+                                    handler_type local_handler{};
+                                    {
+                                        std::scoped_lock lock(ctx->mutex_);
+                                        if (ctx->done_) {
+                                            return;
+                                        }
+                                        --ctx->expected_responses_;
+                                        if (resp.ctx.ec()) {
+                                            if (ctx->expected_responses_ > 0) {
+                                                // just ignore the response
+                                                return;
+                                            }
+                                        } else {
+                                            lookup_in_all_replicas_response::entry top_entry{};
+                                            top_entry.cas = resp.cas;
+                                            top_entry.deleted = resp.deleted;
+                                            top_entry.is_replica = true;
+                                            for (auto& field : resp.fields) {
+                                                lookup_in_all_replicas_response::entry::lookup_in_entry lookup_in_entry{};
+                                                lookup_in_entry.path = field.path;
+                                                lookup_in_entry.value = field.value;
+                                                lookup_in_entry.status = field.status;
+                                                lookup_in_entry.ec = field.ec;
+                                                lookup_in_entry.exists = field.exists;
+                                                lookup_in_entry.original_index = field.original_index;
+                                                lookup_in_entry.opcode = field.opcode;
+                                                top_entry.fields.emplace_back(lookup_in_entry);
+                                            }
+                                            ctx->result_.emplace_back(lookup_in_all_replicas_response::entry{ top_entry });
+                                        }
+                                        if (ctx->expected_responses_ == 0) {
+                                            ctx->done_ = true;
+                                            std::swap(local_handler, ctx->handler_);
+                                        }
+                                    }
+                                    if (local_handler) {
+                                        return local_handler({ std::move(resp.ctx), std::move(ctx->result_) });
+                                    }
+                                });
+              }
+
+              core->execute(lookup_in_request{ document_id{ id }, {}, {}, false, specs, timeout }, [ctx](lookup_in_response&& resp) {
+                  handler_type local_handler{};
+                  {
+                      std::scoped_lock lock(ctx->mutex_);
+                      if (ctx->done_) {
+                          return;
+                      }
+                      --ctx->expected_responses_;
+                      if (resp.ctx.ec()) {
+                          if (ctx->expected_responses_ > 0) {
+                              // just ignore the response
+                              return;
+                          }
+                      } else {
+                          lookup_in_all_replicas_response::entry top_entry{};
+                          top_entry.cas = resp.cas;
+                          top_entry.deleted = resp.deleted;
+                          top_entry.is_replica = false;
+                          for (auto& field : resp.fields) {
+                              lookup_in_all_replicas_response::entry::lookup_in_entry lookup_in_entry{};
+                              lookup_in_entry.path = field.path;
+                              lookup_in_entry.value = field.value;
+                              lookup_in_entry.status = field.status;
+                              lookup_in_entry.ec = field.ec;
+                              lookup_in_entry.exists = field.exists;
+                              lookup_in_entry.original_index = field.original_index;
+                              lookup_in_entry.opcode = field.opcode;
+                              top_entry.fields.emplace_back(lookup_in_entry);
+                          }
+                          ctx->result_.emplace_back(lookup_in_all_replicas_response::entry{ top_entry });
+                      }
+                      if (ctx->expected_responses_ == 0) {
+                          ctx->done_ = true;
+                          std::swap(local_handler, ctx->handler_);
+                      }
+                  }
+                  if (local_handler) {
+                      return local_handler({ std::move(resp.ctx), std::move(ctx->result_) });
+                  }
+              });
+          });
+    }
+};
+
+template<>
+struct is_compound_operation<lookup_in_all_replicas_request> : public std::true_type {
+};
+} // namespace couchbase::core::operations