couchbase 4.0.0 → 4.1.2

package/deps/couchbase-transactions-cxx/include/couchbase/transactions/transaction_config.hxx

@@ -19,6 +19,7 @@
 #include <couchbase/operations/document_query.hxx>
 #include <couchbase/support.hxx>
 #include <couchbase/transactions/durability_level.hxx>
+#include <couchbase/transactions/transaction_keyspace.hxx>
 #include <memory>
 #include <optional>
 
@@ -147,7 +148,7 @@ namespace transactions
          *
          * @param scan_consistency Desired default scan consistency
          */
-        void scan_consistency(couchbase::operations::query_request::scan_consistency_type scan_consistency)
+        void scan_consistency(couchbase::query_scan_consistency scan_consistency)
         {
             scan_consistency_ = scan_consistency;
         }
@@ -156,7 +157,7 @@ namespace transactions
          *
          * @return scan consistency for transactional query operations.
          */
-        CB_NODISCARD operations::query_request::scan_consistency_type scan_consistency() const
+        CB_NODISCARD couchbase::query_scan_consistency scan_consistency() const
         {
             return scan_consistency_;
         }
@@ -205,12 +206,18 @@ namespace transactions
         {
             return cleanup_client_attempts_;
         }
+
+        void custom_metadata_collection(const transaction_keyspace& keyspace)
+        {
+            custom_metadata_collection_ = keyspace;
+        }
+
         void custom_metadata_collection(const std::string& bucket, const std::string& scope, const std::string& collection)
         {
-            custom_metadata_collection_.emplace(bucket, scope, collection, "");
+            custom_metadata_collection_.emplace(bucket, scope, collection);
         }
 
-        CB_NODISCARD std::optional<couchbase::document_id> custom_metadata_collection() const
+        CB_NODISCARD std::optional<transaction_keyspace> custom_metadata_collection() const
         {
             return custom_metadata_collection_;
         }
@@ -218,10 +225,9 @@ namespace transactions
         couchbase::document_id atr_id_from_bucket_and_key(const std::string& bucket, const std::string& key) const
         {
             if (custom_metadata_collection_) {
-                return { custom_metadata_collection_->bucket(),
-                         custom_metadata_collection_->scope(),
-                         custom_metadata_collection_->collection(),
-                         key };
+                return {
+                    custom_metadata_collection_->bucket, custom_metadata_collection_->scope, custom_metadata_collection_->collection, key
+                };
             }
             return { bucket, "_default", "_default", key };
         }
@@ -250,8 +256,8 @@ namespace transactions
         bool cleanup_client_attempts_;
         std::unique_ptr<attempt_context_testing_hooks> attempt_context_hooks_;
         std::unique_ptr<cleanup_testing_hooks> cleanup_hooks_;
-        couchbase::operations::query_request::scan_consistency_type scan_consistency_;
-        std::optional<couchbase::document_id> custom_metadata_collection_;
+        couchbase::query_scan_consistency scan_consistency_;
+        std::optional<transaction_keyspace> custom_metadata_collection_;
     };
 } // namespace transactions
 } // namespace couchbase

package/deps/couchbase-transactions-cxx/include/couchbase/transactions/transaction_keyspace.hxx

@@ -0,0 +1,56 @@
+/*
+ *     Copyright 2021 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+#pragma once
+
+#include <couchbase/document_id.hxx>
+#include <string>
+
+namespace couchbase::transactions
+{
+/**
+ * @brief  Offline, serializable representation of a bucket, scope, and collection
+ */
+struct transaction_keyspace {
+    std::string bucket;
+    std::string scope{ "_default" };
+    std::string collection{ "_default" };
+
+    transaction_keyspace(const std::string& bucket_name, const std::string& scope_name, const std::string& collection_name)
+      : bucket(bucket_name)
+      , scope(scope_name)
+      , collection(collection_name)
+    {
+        if (scope.empty()) {
+            scope = "_default";
+        }
+        if (collection.empty()) {
+            collection = "_default";
+        }
+    }
+
+    transaction_keyspace(const std::string& bucket_name)
+      : transaction_keyspace{ bucket_name, "_default", "_default" }
+    {
+    }
+
+    transaction_keyspace(const couchbase::document_id& id)
+      : bucket(id.bucket())
+      , scope(id.scope())
+      , collection(id.collection())
+    {
+    }
+};
+} // namespace couchbase::transactions

package/deps/couchbase-transactions-cxx/include/couchbase/transactions/transaction_query_options.hxx

@@ -51,13 +51,13 @@ class transaction_query_options
         return *this;
     }
 
-    transaction_query_options& scan_consistency(couchbase::operations::query_request::scan_consistency_type scan_consistency)
+    transaction_query_options& scan_consistency(couchbase::query_scan_consistency scan_consistency)
     {
         query_req_.scan_consistency = scan_consistency;
         return *this;
     }
 
-    transaction_query_options& profile(couchbase::operations::query_request::profile_mode mode)
+    transaction_query_options& profile(couchbase::query_profile_mode mode)
     {
         query_req_.profile = mode;
         return *this;

package/deps/couchbase-transactions-cxx/include/couchbase/transactions.hxx

@@ -21,6 +21,7 @@
 #include <thread>
 
 #include <couchbase/cluster.hxx>
+#include <couchbase/logger/logger.hxx>
 #include <couchbase/transactions/async_attempt_context.hxx>
 #include <couchbase/transactions/attempt_context.hxx>
 #include <couchbase/transactions/exceptions.hxx>
@@ -54,14 +55,15 @@ namespace transactions
     using txn_complete_callback = std::function<void(std::optional<transaction_exception>, std::optional<transaction_result>)>;
 
     /**
-     * @brief logging levels
+     * @brief set log level for transactions
      */
-    enum class log_level { TRACE, DEBUG, INFO, WARN, ERROR, CRITICAL, OFF };
+    void set_transactions_log_level(couchbase::logger::level level);
 
     /**
-     * @brief set log level for transactions
+     * @brief create the loggers, using a custom spdlog::sink, if desired.
      */
-    void set_transactions_log_level(log_level level);
+    void create_loggers(couchbase::logger::level level = couchbase::logger::level::off, spdlog::sink_ptr = nullptr);
+
     /**
      * @mainpage
      * A transaction consists of a lambda containing all the operations you wish to perform within a transaction.

package/deps/couchbase-transactions-cxx/src/transactions/atr_cleanup_entry.cxx

@@ -256,7 +256,7 @@ tx::atr_cleanup_entry::commit_docs(std::shared_ptr<spdlog::logger> logger,
                 }
                 if (doc.links().is_deleted()) {
                     couchbase::operations::insert_request req{ doc.id() };
-                    req.value = content;
+                    req.value = couchbase::utils::to_binary(content);
                     auto barrier = std::make_shared<std::promise<result>>();
                     auto f = barrier->get_future();
                     cleanup_->cluster_ref().execute(wrap_durable_request(req, cleanup_->config(), dl),

package/deps/couchbase-transactions-cxx/src/transactions/attempt_context_impl.cxx

@@ -688,7 +688,7 @@ attempt_context_impl::query_begin_work(Handler&& cb)
     txdata["config"]["numAtrs"] = 1024;
     opts.raw("numatrs", jsonify(1024));
     txdata["config"]["durabilityLevel"] = durability_level_to_string(overall_.config().durability_level());
-    opts.raw("durability_level", jsonify(durability_level_to_string(overall_.config().durability_level())));
+    opts.raw("durability_level", jsonify(durability_level_to_string_for_query(overall_.config().durability_level())));
     if (atr_id_) {
         txdata["atr"] = nlohmann::json::object();
         txdata["atr"]["scp"] = atr_id_->scope();
@@ -725,10 +725,8 @@ attempt_context_impl::query_begin_work(Handler&& cb)
                STAGE_QUERY_BEGIN_WORK,
                true,
                [this, cb = std::move(cb)](std::exception_ptr err, operations::query_response resp) mutable {
-                   if (resp.ctx.last_dispatched_to) {
-                       trace("begin_work setting query node to {}", *resp.ctx.last_dispatched_to);
-                       op_list_.set_query_node(resp.ctx.last_dispatched_to.value());
-                   }
+                   trace("begin_work setting query node to {}", resp.served_by_node);
+                   op_list_.set_query_node(resp.served_by_node);
                    return cb(err);
                });
 }

package/deps/couchbase-transactions-cxx/src/transactions/logging.cxx

@@ -21,9 +21,17 @@ namespace couchbase
 {
 namespace transactions
 {
+    // TODO: consider always using async logger?  Makes life easier, I think, in
+    //       wrappers.  For instance, in python the GIL may, or may not, be held
+    //       by the thread that is logging.   So there are deadlock possibilities
+    //       that can only be worked around by making the custom sink asynchronous.
+    //       A single thread in the thread pool that services each async logger would
+    //       keep ordering in place, _and_ make a much simpler sink to implement.
+    //       That would have to also be done in the client, and of course, there is the
+    //       sizing of the queue to take into account, etc...
     std::shared_ptr<spdlog::logger> init_txn_log()
     {
-        static std::shared_ptr<spdlog::logger> txnlogger = spdlog::stdout_logger_mt(TXN_LOG);
+        static auto txnlogger = spdlog::stdout_logger_mt(TXN_LOG);
         return txnlogger;
     }
 
@@ -38,33 +46,57 @@ namespace transactions
         return txnlogger;
     }
 
-    spdlog::level::level_enum cb_to_spdlog_level(couchbase::transactions::log_level level)
+    // TODO: better integration with client, so we don't need to repeat this private
+    // method.
+    spdlog::level::level_enum translate_level(couchbase::logger::level level)
     {
         switch (level) {
-            case log_level::TRACE:
-                return spdlog::level::trace;
-            case log_level::DEBUG:
-                return spdlog::level::debug;
-            case log_level::INFO:
-                return spdlog::level::info;
-            case log_level::WARN:
-                return spdlog::level::warn;
-            case log_level::ERROR:
-                return spdlog::level::err;
-            case log_level::CRITICAL:
-                return spdlog::level::critical;
-            default:
-                return spdlog::level::off;
+            case couchbase::logger::level::trace:
+                return spdlog::level::level_enum::trace;
+            case couchbase::logger::level::debug:
+                return spdlog::level::level_enum::debug;
+            case couchbase::logger::level::info:
+                return spdlog::level::level_enum::info;
+            case couchbase::logger::level::warn:
+                return spdlog::level::level_enum::warn;
+            case couchbase::logger::level::err:
+                return spdlog::level::level_enum::err;
+            case couchbase::logger::level::critical:
+                return spdlog::level::level_enum::critical;
+            case couchbase::logger::level::off:
+                return spdlog::level::level_enum::off;
         }
+        return spdlog::level::level_enum::trace;
     }
 
-    void set_transactions_log_level(log_level level)
+    void set_transactions_log_level(couchbase::logger::level level)
     {
-        spdlog::level::level_enum lvl = cb_to_spdlog_level(level);
+        spdlog::level::level_enum lvl = translate_level(level);
         txn_log->set_level(lvl);
         attempt_cleanup_log->set_level(lvl);
         lost_attempts_cleanup_log->set_level(lvl);
     }
 
+    // This cannot be done in multiple threads at the same time.   We could
+    // consider a mutex, but eventually we will merge with the cxx_client so
+    // this will be fine for now.   Unsure if this will lead to issues if called
+    // while logging is happening in other threads.  Do this once, at startup.
+    void create_loggers(couchbase::logger::level level, spdlog::sink_ptr sink)
+    {
+        if (nullptr != sink) {
+            sink->set_level(translate_level(level));
+            txn_log->flush();
+            txn_log->sinks().clear();
+            txn_log->sinks().push_back(sink);
+            attempt_cleanup_log->flush();
+            attempt_cleanup_log->sinks().clear();
+            attempt_cleanup_log->sinks().push_back(sink);
+            lost_attempts_cleanup_log->flush();
+            lost_attempts_cleanup_log->sinks().clear();
+            lost_attempts_cleanup_log->sinks().push_back(sink);
+        }
+        set_transactions_log_level(level);
+    }
+
 } // namespace transactions
 } // namespace couchbase

package/deps/couchbase-transactions-cxx/src/transactions/result.hxx

@@ -191,12 +191,16 @@ namespace transactions
         template<typename Resp>
         static result create_from_response(const Resp& resp)
         {
+            std::string content;
+            content.resize(resp.value.size());
+            memcpy(content.data(), resp.value.data(), resp.value.size());
+
             result res{};
             res.ec = resp.ctx.ec;
             res.cas = resp.cas.value;
             res.key = resp.ctx.id.key();
             res.flags = resp.flags;
-            res.raw_value = resp.value;
+            res.raw_value = std::move(content);
             return res;
         }
 

package/deps/couchbase-transactions-cxx/src/transactions/staged_mutation.cxx

@@ -294,7 +294,8 @@ tx::staged_mutation_queue::commit_doc(attempt_context_impl& ctx, staged_mutation
             result res;
             if (item.type() == staged_mutation_type::INSERT && !cas_zero_mode) {
                 couchbase::operations::insert_request req{ item.doc().id() };
-                req.value = item.doc().content<nlohmann::json>().dump();
+                auto content = item.doc().content<nlohmann::json>().dump();
+                req.value = couchbase::utils::to_binary(content);
                 wrap_durable_request(req, ctx.overall_.config());
                 auto barrier = std::make_shared<std::promise<result>>();
                 auto f = barrier->get_future();

package/deps/couchbase-transactions-cxx/src/transactions/transaction_config.cxx

@@ -31,7 +31,7 @@ namespace transactions
       , cleanup_client_attempts_(true)
       , attempt_context_hooks_(new attempt_context_testing_hooks())
       , cleanup_hooks_(new cleanup_testing_hooks())
-      , scan_consistency_(operations::query_request::scan_consistency_type::request_plus)
+      , scan_consistency_(couchbase::query_scan_consistency::request_plus)
     {
     }
 

package/deps/couchbase-transactions-cxx/src/transactions/transactions.cxx

@@ -30,6 +30,30 @@ tx::transactions::transactions(cluster& cluster, const transaction_config& confi
   , cleanup_(new transactions_cleanup(cluster_, config_))
 {
     txn_log->info("couchbase transactions {}{} creating new transaction object", VERSION_STR, VERSION_SHA);
+    // if the config specifies custom metadata collection, lets be sure to open that bucket
+    // on the cluster before we start.  NOTE: we actually do call get_and_open_buckets which opens all the buckets
+    // on the cluster (that we have permissions to open) in the cleanup.   However, that is happening asynchronously
+    // so there's a chance we will fail to have opened the custom metadata collection bucket before trying to make a
+    // transaction.   We have to open this one _now_.
+    if (config_.custom_metadata_collection()) {
+        auto barrier = std::make_shared<std::promise<std::error_code>>();
+        auto f = barrier->get_future();
+        std::atomic<bool> callback_called{ false };
+        cluster_.open_bucket(config_.custom_metadata_collection()->bucket, [&callback_called, barrier](std::error_code ec) {
+            if (callback_called.load()) {
+                return;
+            }
+            callback_called = true;
+            barrier->set_value(ec);
+        });
+        auto err = f.get();
+        if (err) {
+            auto err_msg = fmt::format("error opening custom_metadata_collection bucket '{}' specified in the config!",
+                                       config_.custom_metadata_collection()->bucket);
+            txn_log->error(err_msg);
+            throw std::runtime_error(err_msg);
+        }
+    }
 }
 
 tx::transactions::~transactions() = default;

package/deps/couchbase-transactions-cxx/tests/transactions/logging_t.cpp

@@ -0,0 +1,127 @@
+/*
+ *     Copyright 2022 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "helpers.hxx"
+#include "transactions_env.h"
+#include <couchbase/errors.hxx>
+#include <couchbase/transactions/internal/logging.hxx>
+#include <filesystem>
+#include <fstream>
+#include <gtest/gtest.h>
+#include <iostream>
+#include <iterator>
+#include <spdlog/fwd.h>
+#include <spdlog/sinks/base_sink.h>
+
+using namespace couchbase::transactions;
+
+auto doc_content = nlohmann::json::parse("{\"some\": \"thing\"}");
+
+class TrivialFileSink : public spdlog::sinks::base_sink<std::mutex>
+{
+  protected:
+    void sink_it_(const spdlog::details::log_msg& msg) override
+    {
+        spdlog::memory_buf_t formatted;
+        base_sink<std::mutex>::formatter_->format(msg, formatted);
+        std::cerr << formatted.data();
+    }
+    void flush_() override
+    {
+    }
+
+  private:
+};
+
+TEST(LoggingTests, CanLogToStdoutByDefault)
+{
+    couchbase::logger::create_blackhole_logger();
+    std::string log_message = "I am a log";
+    couchbase::transactions::set_transactions_log_level(couchbase::logger::level::debug);
+    testing::internal::CaptureStdout();
+    txn_log->debug(log_message);
+    txn_log->flush();
+    auto out = testing::internal::GetCapturedStdout();
+    ASSERT_FALSE(out.empty());
+    ASSERT_NE(std::string::npos, out.find(log_message));
+    couchbase::logger::create_console_logger();
+}
+
+TEST(LoggingTests, LogLevelsWork)
+{
+    couchbase::logger::create_blackhole_logger();
+    std::string log_message = "I am a log";
+    couchbase::transactions::set_transactions_log_level(couchbase::logger::level::info);
+    testing::internal::CaptureStdout();
+    txn_log->debug(log_message);
+    txn_log->flush();
+    auto out = testing::internal::GetCapturedStdout();
+    ASSERT_TRUE(out.empty());
+    couchbase::logger::create_console_logger();
+}
+
+TEST(LoggingTests, CanUseCustomSink)
+{
+    couchbase::logger::create_blackhole_logger();
+    std::string log_message = "I am a log";
+    auto sink = std::make_shared<TrivialFileSink>();
+    couchbase::transactions::create_loggers(couchbase::logger::level::debug, sink);
+    testing::internal::CaptureStdout();
+    testing::internal::CaptureStderr();
+    txn_log->debug(log_message);
+    txn_log->flush();
+    auto out = testing::internal::GetCapturedStdout();
+    ASSERT_TRUE(out.empty()) << "out = " << out;
+    auto err = testing::internal::GetCapturedStderr();
+    ASSERT_FALSE(err.empty());
+    ASSERT_NE(std::string::npos, err.find(log_message)) << "err = " << err;
+    couchbase::logger::create_console_logger();
+}
+
+TEST(LoggingTests, CustomSinkRespectsLogLevels)
+{
+    couchbase::logger::create_blackhole_logger();
+    std::string log_message = "I am a log";
+    auto sink = std::make_shared<TrivialFileSink>();
+    couchbase::transactions::create_loggers(couchbase::logger::level::info, sink);
+    testing::internal::CaptureStdout();
+    testing::internal::CaptureStderr();
+    txn_log->debug(log_message);
+    txn_log->flush();
+    auto out = testing::internal::GetCapturedStdout();
+    ASSERT_TRUE(out.empty()) << "out = " << out;
+    auto err = testing::internal::GetCapturedStderr();
+    ASSERT_TRUE(err.empty()) << "err = " << err;
+    couchbase::logger::create_console_logger();
+}
+
+TEST(LoggingTests, CustomSinkRespectsLogLevelChanges)
+{
+    couchbase::logger::create_blackhole_logger();
+    std::string log_message = "I am a log";
+    auto sink = std::make_shared<TrivialFileSink>();
+    couchbase::transactions::create_loggers(couchbase::logger::level::debug, sink);
+    couchbase::transactions::set_transactions_log_level(couchbase::logger::level::info);
+    testing::internal::CaptureStdout();
+    testing::internal::CaptureStderr();
+    txn_log->debug(log_message);
+    txn_log->flush();
+    auto out = testing::internal::GetCapturedStdout();
+    ASSERT_TRUE(out.empty()) << "out = " << out;
+    auto err = testing::internal::GetCapturedStderr();
+    ASSERT_TRUE(err.empty()) << "err = " << err;
+    couchbase::logger::create_console_logger();
+}