couchbase 3.3.0 → 3.4.0

data/ext/couchbase/core/transactions/uid_generator.cxx

@@ -0,0 +1,27 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "core/platform/uuid.h"
+
+#include "uid_generator.hxx"
+
+// TODO: Remove this, and just use client directly
+
+std::string
+couchbase::core::transactions::uid_generator::next()
+{
+    return core::uuid::to_string(core::uuid::random());
+}

data/ext/couchbase/core/transactions/uid_generator.hxx

@@ -0,0 +1,29 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <mutex>
+#include <string>
+
+namespace couchbase::core::transactions
+{
+class uid_generator
+{
+  public:
+    static std::string next();
+};
+} // namespace couchbase::core::transactions

data/ext/couchbase/core/transactions/utils.cxx

@@ -0,0 +1,123 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "core/operations.hxx"
+#include "core/operations/management/bucket_get_all.hxx"
+
+#include "internal/utils.hxx"
+
+namespace couchbase::core::transactions
+{
+std::string
+collection_spec_from_id(const core::document_id& id)
+{
+    std::string retval = id.scope();
+    return retval.append(".").append(id.collection());
+}
+
+bool
+document_ids_equal(const core::document_id& id1, const core::document_id& id2)
+{
+    return id1.key() == id2.key() && id1.bucket() == id2.bucket() && id1.scope() == id2.scope() && id1.collection() == id2.collection();
+}
+
+std::string
+jsonify(const tao::json::value& obj)
+{
+    return core::utils::json::generate(obj);
+}
+
+std::uint64_t
+now_ns_from_vbucket(const tao::json::value& vbucket)
+{
+    std::string now_str = vbucket.at("HLC").at("now").get_string();
+    return stoull(now_str, nullptr, 10) * 1000000000;
+}
+
+void
+wrap_collection_call(result& res, std::function<void(result&)> call)
+{
+    call(res);
+    if (!res.is_success()) {
+        throw client_error(res);
+    }
+    if (!res.values.empty() && !res.ignore_subdoc_errors) {
+        for (const auto& v : res.values) {
+            if (v.status != subdoc_result::status_type::success) {
+                throw client_error(res);
+            }
+        }
+    }
+}
+
+result
+wrap_operation_future(std::future<result>& fut, bool ignore_subdoc_errors)
+{
+    auto res = fut.get();
+    if (!res.is_success()) {
+        throw client_error(res);
+    }
+    // we should raise here, as we are doing a non-subdoc request and can't specify
+    // access_deleted.  TODO: consider changing client to return document_not_found
+    if (res.is_deleted && res.values.empty()) {
+        res.ec = couchbase::errc::key_value::document_not_found;
+        throw client_error(res);
+    }
+    if (!res.values.empty() && !ignore_subdoc_errors) {
+        for (const auto& v : res.values) {
+            if (v.status != subdoc_result::status_type::success) {
+                throw client_error(res);
+            }
+        }
+    }
+    return res;
+}
+
+template<>
+bool
+is_error(const core::operations::mutate_in_response& resp)
+{
+    return resp.ctx.ec() || resp.ctx.first_error_index();
+}
+
+template<>
+std::optional<error_class>
+error_class_from_response_extras(const core::operations::mutate_in_response& resp)
+{
+    if (!resp.ctx.first_error_index()) {
+        return {};
+    }
+    auto status = resp.fields.at(*resp.ctx.first_error_index()).status;
+    if (status == key_value_status_code::subdoc_path_not_found) {
+        return FAIL_PATH_NOT_FOUND;
+    }
+    if (status == key_value_status_code::subdoc_path_exists) {
+        return FAIL_PATH_ALREADY_EXISTS;
+    }
+    return FAIL_OTHER;
+}
+
+core::document_id
+atr_id_from_bucket_and_key(const couchbase::transactions::transactions_config::built& cfg,
+                           const std::string& bucket,
+                           const std::string& key)
+{
+    if (cfg.metadata_collection) {
+        return { cfg.metadata_collection->bucket, cfg.metadata_collection->scope, cfg.metadata_collection->collection, key };
+    }
+    return { bucket, couchbase::scope::default_name, couchbase::collection::default_name, key };
+}
+} // namespace couchbase::core::transactions

data/ext/couchbase/core/transactions/waitable_op_list.hxx

@@ -0,0 +1,189 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include "internal/logging.hxx"
+
+#include <condition_variable>
+#include <mutex>
+
+namespace couchbase::core::transactions
+{
+
+class async_operation_conflict : public std::runtime_error
+{
+  public:
+    async_operation_conflict(const std::string& msg)
+      : std::runtime_error(msg)
+    {
+    }
+};
+struct attempt_mode {
+    enum class modes { KV, QUERY };
+    modes mode;
+    std::string query_node;
+
+    attempt_mode()
+      : mode(modes::KV)
+    {
+    }
+
+    bool is_query()
+    {
+        return mode == modes::QUERY;
+    }
+};
+
+class waitable_op_list
+{
+  public:
+    waitable_op_list()
+      : count_(0)
+      , allow_ops_(true)
+      , mode_()
+      , in_flight_(0)
+    {
+    }
+
+    void increment_ops()
+    {
+        change_count(1);
+    }
+    void decrement_ops()
+    {
+        change_count(-1);
+    }
+    void wait_and_block_ops()
+    {
+        std::unique_lock<std::mutex> lock(mutex_);
+        cv_ops_.wait(lock, [this]() { return (0 == count_); });
+        // we have the lock.  Block all further ops
+        allow_ops_ = false;
+    }
+    attempt_mode get_mode()
+    {
+        std::unique_lock<std::mutex> lock(mutex_);
+        if (mode_.mode == attempt_mode::modes::KV) {
+            return {};
+        }
+        if (!mode_.query_node.empty()) {
+            return mode_;
+        }
+        // Another op has set the query_mode_, and hasn't set the
+        // query_node_ yet.   So we wait.
+        cv_query_.wait(lock, [this]() { return !mode_.query_node.empty(); });
+        return mode_;
+    }
+
+    template<typename BeginWorkHandler, typename DoQueryHandler>
+    void set_query_mode(BeginWorkHandler&& begin_work_cb, DoQueryHandler&& cb)
+    {
+        std::unique_lock<std::mutex> lock(mutex_);
+        // called within an op, so decrement in_flight from that op, wait for
+        // other in_flight to complete.
+        in_flight_--;
+        if (mode_.mode == attempt_mode::modes::KV) {
+            // wait until all in_flight ops are done
+            CB_TXN_LOG_TRACE("set_query_mode: waiting for in_flight ops to go to 0...");
+            cv_in_flight_.wait(lock, [this]() { return (0 == in_flight_); });
+            // ok, now no outstanding ops(apart from the query that called this), and I have the lock, so...
+            if (mode_.mode == attempt_mode::modes::KV) {
+                CB_TXN_LOG_TRACE("set_query_mode: in_flight ops = 0, we were kv, setting mode to query");
+                // still kv, so now (while blocking) set the mode
+                mode_.mode = attempt_mode::modes::QUERY;
+                // ok to unlock now, as any racing set_query_mode will wait for
+                // a node to be set.
+                in_flight_++;
+                lock.unlock();
+                // now (outside the lock), call the callback, which does the begin_work
+                // when initially setting query mode.
+                begin_work_cb();
+                return;
+            }
+        }
+        // you make it here, and someone else is currently setting the node (a byproduct of
+        // calling the callback).  So wait for that.
+        CB_TXN_LOG_TRACE("set_query_mode: mode already query, waiting for node to be set...");
+        cv_query_.wait(lock, [this]() { return !mode_.query_node.empty(); });
+        in_flight_++;
+        CB_TXN_LOG_TRACE("set_query_mode: node set, continuing...");
+        lock.unlock();
+        cb();
+    }
+    void reset_query_mode()
+    {
+        // when begin work errors out, it is fatal, so reset to kv mode here, allowing
+        // rollback to function properly.
+        std::unique_lock<std::mutex> lock;
+        mode_.mode = attempt_mode::modes::KV;
+        cv_query_.notify_all();
+    }
+
+    void set_query_node(const std::string& node)
+    {
+        std::unique_lock<std::mutex> lock(mutex_);
+        assert(mode_.mode == attempt_mode::modes::QUERY);
+        mode_.query_node = node;
+        // now notify everyone waiting in get_mode()
+        cv_query_.notify_all();
+    }
+    void decrement_in_flight()
+    {
+        std::lock_guard<std::mutex> lock(mutex_);
+        in_flight_--;
+        CB_TXN_LOG_TRACE("in_flight decremented to {}", in_flight_);
+        assert(in_flight_ >= 0);
+        if (0 == in_flight_) {
+            cv_in_flight_.notify_all();
+        }
+    }
+
+  private:
+    void change_count(int32_t val)
+    {
+        std::lock_guard<std::mutex> lock(mutex_);
+        if (allow_ops_) {
+            count_ += val;
+            if (val > 0) {
+                in_flight_ += val;
+            }
+            CB_TXN_LOG_TRACE("op count changed by {} to {}, {} in_flight", val, count_, in_flight_);
+            assert(count_ >= 0);
+            assert(in_flight_ >= 0);
+            if (0 == count_) {
+                cv_ops_.notify_all();
+            }
+            if (0 == in_flight_) {
+                cv_in_flight_.notify_all();
+            }
+        } else {
+            CB_TXN_LOG_ERROR("operation attempted after commit/rollback");
+            throw async_operation_conflict("Operation attempted after commit or rollback");
+        }
+    }
+
+  private:
+    int32_t count_;
+    bool allow_ops_;
+    attempt_mode mode_;
+    int32_t in_flight_;
+    std::condition_variable cv_ops_;
+    std::condition_variable cv_query_;
+    std::condition_variable cv_in_flight_;
+    std::mutex mutex_;
+};
+} // namespace couchbase::core::transactions

data/ext/couchbase/core/transactions.hxx

@@ -0,0 +1,255 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <couchbase/transactions.hxx>
+
+#include "couchbase/transactions/transaction_options.hxx"
+#include "couchbase/transactions/transaction_result.hxx"
+#include "couchbase/transactions/transactions_config.hxx"
+#include "transactions/async_attempt_context.hxx"
+#include "transactions/attempt_context.hxx"
+#include "transactions/exceptions.hxx"
+
+#include "core/logger/logger.hxx"
+
+#include <spdlog/common.h>
+
+#include <cmath>
+#include <functional>
+#include <thread>
+
+// workaround for MSVC define overlap with log levels
+#undef ERROR
+
+/**
+ * @file
+ * Main header file for Couchbase Transactions
+ */
+
+namespace couchbase::core
+{
+class cluster;
+
+namespace transactions
+{
+/** @internal
+ */
+class transactions_cleanup;
+
+/** @brief Transaction logic should be contained in a lambda of this form */
+using logic = std::function<void(attempt_context&)>;
+
+/** @brief AsyncTransaction logic should be contained in a lambda of this form */
+using async_logic = std::function<void(async_attempt_context&)>;
+
+/** @brief AsyncTransaction callback when transaction has completed */
+using txn_complete_callback =
+  std::function<void(std::optional<transaction_exception>, std::optional<::couchbase::transactions::transaction_result>)>;
+
+/**
+ * @mainpage
+ * A transaction consists of a lambda containing all the operations you wish to perform within a transaction.
+ * The @ref transactions.run() call yields an @ref attempt_context which you use for those operations.
+ *
+ * @section txn_overview Overview
+ *
+ * A very simple transaction:
+ *
+ * @code{.cpp}
+ * cluster c("couchbase://127.0.0.1", "Administrator", "password");
+ * transactions_config config;
+ * config.durability_level(transactions::durability_level::MAJORITY);
+ * auto b = cluster.bucket("default");
+ * auto coll = b->default_collection();
+ * transactions txn(c, config);
+ *
+ * try {
+ *     txn.run([&](transactions::attempt_context& ctx) {
+ *         ctx.upsert(coll, "somekey", nlohmann::json::parse("{\"a\":\"thing\"}"));
+ *         ctx.insert(coll, "someotherkey", nlohmann::json::parse("{"\a\":\"different thing\"}"));
+ *     });
+ *     cout << "txn successful" << endl;
+ * } catch (const transaction_failed& failed) {
+ *     cerr << "txn failed: " << failed.what() << endl;
+ * } catch (const transaction_expired& expired) {
+ *    cerr << "txn timed out" << expired.what() << endl;
+ * }
+ * @endcode
+ *
+ * This upserts a document, and inserts another into the default collection in the bucket named "default".  If
+ * unsuccessful, an exception will be raised.
+ *
+ * For a much more detailed example, see @ref examples/game_server.cxx
+ *
+ * @section txn_best_practices Best Practices
+ *
+ * Each @ref transactions instances spins up background threads to perform cleanup of metadata
+ * that could be left behind after failed transactions.  For that reason, creating many
+ * @ref transactions objects, especially if they are long-lived, will consume resources.  We recommend
+ * simply creating one @ref transactions object per process, and using that for the life of the process,
+ * when possible.
+ *
+ * The @ref transactions, @ref cluster, @ref bucket, and @ref collection instances are all safe to use
+ * across threads.  The @ref cluster and @ref bucket use libcouchbase internally, and manage a pool of
+ * libcouchbase instances, since those instances are not to be used across threads.  The maximum number
+ * of instances in the pools (for either the @ref cluster or @ref bucket) can be specified at the time
+ * you construct the @ref cluster.  See @ref cluster_options for details.
+ *
+ * Since a @ref transactions instance spins up one thread per bucket for background cleanup tasks, a reasonable
+ * starting point for sizing the @ref cluster_options::max_bucket_instances would be one per bucket, plus
+ * one per thread that uses the @ref cluster (or the @ref transactions object constructed with that @ref cluster).
+ * This could be tuned lower potentially, but this is a the maxmimum one should need.  The pool only
+ * creates a new instance when it needs one, so the actual number created may never hit the max.  When
+ * the maximum is reached, the thread will block until one is available.
+ *
+ * @example examples/game_server.cxx
+ */
+
+/** @brief Main class for creating a transaction
+ *
+ */
+class transactions : public couchbase::transactions::transactions
+{
+  public:
+    /**
+     * @brief Create a transactions object.
+     *
+     * Creates a transactions object, which can be used to run transactions within the current thread.
+     *
+     * @param cluster The cluster to use for the transactions.
+     * @param config The configuration parameters to use for the transactions.
+     */
+    transactions(std::shared_ptr<core::cluster> cluster, const couchbase::transactions::transactions_config::built& config);
+    transactions(std::shared_ptr<core::cluster> cluster, const couchbase::transactions::transactions_config& config);
+
+    /**
+     * @brief Destructor
+     */
+    ~transactions();
+
+    /**
+     * @brief Run a transaction
+     *
+     * Expects a lambda, which it calls with an @ref attempt_context reference to be used in the lambda for
+     * the transaction operations.
+     *
+     * @param logic The lambda containing the transaction logic.
+     * @return A struct containing some internal state information about the transaction.
+     * @throws @ref transaction_failed, @ref transaction_expired, @ref transaction_commit_ambiguous, all of which
+     *         share a common base class @ref transaction_exception.
+     */
+    couchbase::transactions::transaction_result run(logic&& code);
+
+    couchbase::transactions::transaction_result run(const couchbase::transactions::transaction_options& config, logic&& code);
+
+    std::pair<couchbase::transaction_error_context, couchbase::transactions::transaction_result> run(
+      ::couchbase::transactions::txn_logic&& code,
+      const couchbase::transactions::transaction_options& cfg = couchbase::transactions::transaction_options()) override;
+
+    /**
+     * @brief Run a transaction
+     *
+     * Expects a lambda, which it calls with an @ref async_attempt_context reference to be used in the lambda for
+     * the asynchronous transaction operations. Upon completion, calls the callback.
+     *
+     * @param logic The lambda containing the async transaction logic.
+     * @param cb Called when the transaction is complete.
+     * @return A struct containing some internal state information about the transaction.
+     * @throws @ref transaction_failed, @ref transaction_expired, @ref transaction_commit_ambiguous, all of which
+     *         share a common base class @ref transaction_exception.
+     */
+    void run(async_logic&& code, txn_complete_callback&& cb);
+
+    void run(const couchbase::transactions::transaction_options& config, async_logic&& code, txn_complete_callback&& cb);
+
+    void run(couchbase::transactions::async_txn_logic&& code,
+             couchbase::transactions::async_txn_complete_logic&& complete_cb,
+             const couchbase::transactions::transaction_options& cfg = {}) override;
+    /**
+     * @internal
+     * called internally - will likely move
+     */
+    void commit(attempt_context& ctx)
+    {
+        ctx.commit();
+    }
+
+    /**
+     * @internal
+     * called internally - will likely move
+     */
+    void rollback(attempt_context& ctx)
+    {
+        ctx.rollback();
+    }
+
+    /**
+     * @brief Shut down the transactions object
+     *
+     * The transaction object cannot be used after this call.  Called in destructor, but
+     * available to call sooner if needed.
+     */
+    void close();
+
+    /**
+     * @brief Return reference to @ref transactions_config.
+     *
+     * @return config for this transactions instance.
+     */
+    [[nodiscard]] couchbase::transactions::transactions_config::built& config()
+    {
+        return config_;
+    }
+
+    /**
+     * @internal
+     * Called internally
+     */
+    [[nodiscard]] const transactions_cleanup& cleanup() const
+    {
+        return *cleanup_;
+    }
+
+    /**
+     * @internal
+     * Called internally
+     */
+    [[nodiscard]] transactions_cleanup& cleanup()
+    {
+        return *cleanup_;
+    }
+
+    /**
+     * @brief Return a reference to the @ref core::cluster
+     *
+     * @return Ref to the cluster used by this transaction object.
+     */
+    [[nodiscard]] std::shared_ptr<core::cluster> cluster_ref()
+    {
+        return cluster_;
+    }
+
+  private:
+    std::shared_ptr<core::cluster> cluster_;
+    couchbase::transactions::transactions_config::built config_;
+    std::unique_ptr<transactions_cleanup> cleanup_;
+    const std::size_t max_attempts_{ 1000 };
+    const std::chrono::milliseconds min_retry_delay_{ 1 };
+};
+} // namespace transactions
+} // namespace couchbase::core

data/ext/couchbase/core/utils/binary.cxx

@@ -0,0 +1,27 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2020-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "binary.hxx"
+
+namespace couchbase::core::utils
+{
+binary
+to_binary(std::string_view value) noexcept
+{
+    return to_binary(value.data(), value.size());
+}
+} // namespace couchbase::core::utils