couchbase 3.4.3 → 3.4.5

data/ext/couchbase/core/operations/document_lookup_in_any_replica.hxx

@@ -0,0 +1,188 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2020-2021 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include "core/error_context/key_value.hxx"
+#include "core/impl/lookup_in_replica.hxx"
+#include "core/impl/subdoc/command.hxx"
+#include "core/operations/document_lookup_in.hxx"
+#include "core/operations/operation_traits.hxx"
+#include "core/utils/movable_function.hxx"
+#include "couchbase/codec/encoded_value.hxx"
+#include "couchbase/error_codes.hxx"
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+namespace couchbase::core::operations
+{
+struct lookup_in_any_replica_response {
+    struct entry {
+        std::string path;
+        codec::binary value;
+        std::size_t original_index;
+        bool exists;
+        protocol::subdoc_opcode opcode;
+        key_value_status_code status;
+        std::error_code ec{};
+    };
+    subdocument_error_context ctx{};
+    couchbase::cas cas{};
+    std::vector<entry> fields{};
+    bool deleted{ false };
+    bool is_replica{ true };
+};
+
+struct lookup_in_any_replica_request {
+    using response_type = lookup_in_any_replica_response;
+    using encoded_request_type = core::protocol::client_request<core::protocol::lookup_in_replica_request_body>;
+    using encoded_response_type = core::protocol::client_response<core::protocol::lookup_in_replica_response_body>;
+
+    core::document_id id;
+    std::vector<couchbase::core::impl::subdoc::command> specs{};
+    std::optional<std::chrono::milliseconds> timeout{};
+    std::shared_ptr<couchbase::tracing::request_span> parent_span{ nullptr };
+
+    template<typename Core, typename Handler>
+    void execute(Core core, Handler handler)
+    {
+        core->with_bucket_configuration(
+          id.bucket(),
+          [core, id = id, timeout = timeout, specs = specs, parent_span = parent_span, h = std::forward<Handler>(handler)](
+            std::error_code ec, const topology::configuration& config) mutable {
+              if (!config.supports_subdoc_read_replica()) {
+                  ec = errc::common::feature_not_available;
+              }
+              if (specs.size() > 16) {
+                  ec = errc::common::invalid_argument;
+              }
+
+              if (ec) {
+                  std::optional<std::string> first_error_path{};
+                  std::optional<std::size_t> first_error_index{};
+                  return h(response_type{
+                    make_subdocument_error_context(make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) });
+              }
+              using handler_type = utils::movable_function<void(response_type)>;
+
+              struct replica_context {
+                  replica_context(handler_type&& handler, std::uint32_t expected_responses)
+                    : handler_(std::move(handler))
+                    , expected_responses_(expected_responses)
+                  {
+                  }
+
+                  handler_type handler_;
+                  std::uint32_t expected_responses_;
+                  bool done_{ false };
+                  std::mutex mutex_{};
+              };
+              auto ctx = std::make_shared<replica_context>(std::move(h), config.num_replicas.value_or(0U) + 1U);
+
+              for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) {
+                  document_id replica_id{ id };
+                  replica_id.node_index(idx);
+                  core->execute(impl::lookup_in_replica_request{ std::move(replica_id), specs, timeout, parent_span },
+                                [ctx](impl::lookup_in_replica_response&& resp) {
+                                    handler_type local_handler;
+                                    {
+                                        std::scoped_lock lock(ctx->mutex_);
+                                        if (ctx->done_) {
+                                            return;
+                                        }
+                                        --ctx->expected_responses_;
+                                        if (resp.ctx.ec()) {
+                                            if (ctx->expected_responses_ > 0) {
+                                                // just ignore the response
+                                                return;
+                                            }
+                                            // consider document irretrievable and give up
+                                            resp.ctx.override_ec(errc::key_value::document_irretrievable);
+                                        }
+                                        ctx->done_ = true;
+                                        std::swap(local_handler, ctx->handler_);
+                                    }
+                                    if (local_handler) {
+                                        response_type res{};
+                                        res.ctx = resp.ctx;
+                                        res.cas = resp.cas;
+                                        res.deleted = resp.deleted;
+                                        res.is_replica = true;
+                                        for (auto& field : resp.fields) {
+                                            auto lookup_in_entry = lookup_in_any_replica_response::entry{};
+                                            lookup_in_entry.path = field.path;
+                                            lookup_in_entry.value = field.value;
+                                            lookup_in_entry.status = field.status;
+                                            lookup_in_entry.ec = field.ec;
+                                            lookup_in_entry.exists = field.exists;
+                                            lookup_in_entry.original_index = field.original_index;
+                                            lookup_in_entry.opcode = field.opcode;
+                                            res.fields.emplace_back(lookup_in_entry);
+                                        }
+                                        return local_handler(res);
+                                    }
+                                });
+              }
+              core->execute(lookup_in_request{ id, {}, {}, false, specs, timeout }, [ctx](lookup_in_response&& resp) {
+                  handler_type local_handler{};
+                  {
+                      std::scoped_lock lock(ctx->mutex_);
+                      if (ctx->done_) {
+                          return;
+                      }
+                      --ctx->expected_responses_;
+                      if (resp.ctx.ec()) {
+                          if (ctx->expected_responses_ > 0) {
+                              // just ignore the response
+                              return;
+                          }
+                          // consider document irretrievable and give up
+                          resp.ctx.override_ec(errc::key_value::document_irretrievable);
+                      }
+                      ctx->done_ = true;
+                      std::swap(local_handler, ctx->handler_);
+                  }
+                  if (local_handler) {
+                      auto res = response_type{};
+                      res.ctx = resp.ctx;
+                      res.cas = resp.cas;
+                      res.deleted = resp.deleted;
+                      res.is_replica = false;
+                      for (auto& field : resp.fields) {
+                          auto lookup_in_entry = lookup_in_any_replica_response::entry{};
+                          lookup_in_entry.path = field.path;
+                          lookup_in_entry.value = field.value;
+                          lookup_in_entry.status = field.status;
+                          lookup_in_entry.ec = field.ec;
+                          lookup_in_entry.exists = field.exists;
+                          lookup_in_entry.original_index = field.original_index;
+                          lookup_in_entry.opcode = field.opcode;
+                          res.fields.emplace_back(lookup_in_entry);
+                      }
+                      return local_handler(res);
+                  }
+              });
+          });
+    }
+};
+
+template<>
+struct is_compound_operation<lookup_in_any_replica_request> : public std::true_type {
+};
+} // namespace couchbase::core::operations

data/ext/couchbase/core/operations/document_query.cxx

@@ -85,6 +85,17 @@ query_request::encode_to(query_request::encoded_request_type& encoded, http_cont
         case couchbase::query_profile::off:
             break;
     }
+    if (use_replica.has_value()) {
+        if (context.config.supports_read_from_replica()) {
+            if (use_replica.value()) {
+                body["use_replica"] = "on";
+            } else {
+                body["use_replica"] = "off";
+            }
+        } else {
+            return errc::common::feature_not_available;
+        }
+    }
     if (max_parallelism) {
         body["max_parallelism"] = std::to_string(max_parallelism.value());
     }

data/ext/couchbase/core/operations/document_query.hxx

@@ -91,6 +91,7 @@ struct query_request {
     bool flex_index{ false };
     bool preserve_expiry{ false };
 
+    std::optional<bool> use_replica{};
     std::optional<std::uint64_t> max_parallelism{};
     std::optional<std::uint64_t> scan_cap{};
     std::optional<std::chrono::milliseconds> scan_wait{};

data/ext/couchbase/core/operations/management/CMakeLists.txt

@@ -28,6 +28,7 @@ add_library(
         collection_create.cxx
         collection_drop.cxx
         collections_manifest_get.cxx
+        collection_update.cxx
         error_utils.cxx
         eventing_deploy_function.cxx
         eventing_drop_function.cxx

data/ext/couchbase/core/operations/management/bucket_create.cxx

@@ -48,17 +48,35 @@ bucket_create_request::encode_to(encoded_request_type& encoded, http_context& /*
         case couchbase::core::management::cluster::bucket_type::unknown:
             break;
     }
-    encoded.body.append(fmt::format("&ramQuotaMB={}", bucket.ram_quota_mb));
-    if (bucket.bucket_type != couchbase::core::management::cluster::bucket_type::memcached) {
-        encoded.body.append(fmt::format("&replicaNumber={}", bucket.num_replicas));
+    if (bucket.ram_quota_mb == 0) {
+        encoded.body.append(fmt::format("&ramQuotaMB={}", 100)); // If not explicitly set, set to prior default value of 100
+    } else {
+        encoded.body.append(fmt::format("&ramQuotaMB={}", bucket.ram_quota_mb));
     }
-    if (bucket.max_expiry > 0) {
-        encoded.body.append(fmt::format("&maxTTL={}", bucket.max_expiry));
+
+    if (bucket.bucket_type != couchbase::core::management::cluster::bucket_type::memcached && bucket.num_replicas.has_value()) {
+        encoded.body.append(fmt::format("&replicaNumber={}", bucket.num_replicas.value()));
+    }
+    if (bucket.max_expiry.has_value()) {
+        encoded.body.append(fmt::format("&maxTTL={}", bucket.max_expiry.value()));
+    }
+    if (bucket.bucket_type != couchbase::core::management::cluster::bucket_type::ephemeral && bucket.replica_indexes.has_value()) {
+        encoded.body.append(fmt::format("&replicaIndex={}", bucket.replica_indexes.value() ? "1" : "0"));
+    }
+    if (bucket.history_retention_collection_default.has_value()) {
+        encoded.body.append(
+          fmt::format("&historyRetentionCollectionDefault={}", bucket.history_retention_collection_default.value() ? "true" : "false"));
     }
-    if (bucket.bucket_type != couchbase::core::management::cluster::bucket_type::ephemeral) {
-        encoded.body.append(fmt::format("&replicaIndex={}", bucket.replica_indexes ? "1" : "0"));
+    if (bucket.history_retention_bytes.has_value()) {
+        encoded.body.append(fmt::format("&historyRetentionBytes={}", bucket.history_retention_bytes.value()));
     }
-    encoded.body.append(fmt::format("&flushEnabled={}", bucket.flush_enabled ? "1" : "0"));
+    if (bucket.history_retention_duration.has_value()) {
+        encoded.body.append(fmt::format("&historyRetentionSeconds={}", bucket.history_retention_duration.value()));
+    }
+    if (bucket.flush_enabled.has_value()) {
+        encoded.body.append(fmt::format("&flushEnabled={}", bucket.flush_enabled.value() ? "1" : "0"));
+    }
+
     switch (bucket.eviction_policy) {
         case couchbase::core::management::cluster::bucket_eviction_policy::full:
             encoded.body.append("&evictionPolicy=fullEviction");
@@ -101,7 +119,7 @@ bucket_create_request::encode_to(encoded_request_type& encoded, http_context& /*
         case couchbase::core::management::cluster::bucket_conflict_resolution::unknown:
             break;
     }
-    if (bucket.minimum_durability_level) {
+    if (bucket.minimum_durability_level.has_value()) {
         switch (bucket.minimum_durability_level.value()) {
             case durability_level::none:
                 encoded.body.append("&durabilityMinLevel=none");
@@ -151,6 +169,9 @@ bucket_create_request::make_response(error_context::http&& ctx, const encoded_re
                 if (errors != nullptr) {
                     std::vector<std::string> error_list{};
                     for (const auto& [code, message] : errors->get_object()) {
+                        if (message.get_string().find("Bucket with given name already exists") != std::string::npos) {
+                            response.ctx.ec = errc::management::bucket_exists;
+                        }
                         error_list.emplace_back(message.get_string());
                     }
                     if (!error_list.empty()) {

data/ext/couchbase/core/operations/management/bucket_update.cxx

@@ -33,13 +33,34 @@ bucket_update_request::encode_to(encoded_request_type& encoded, http_context& /*
     encoded.path = fmt::format("/pools/default/buckets/{}", bucket.name);
 
     encoded.headers["content-type"] = "application/x-www-form-urlencoded";
-    encoded.body.append(fmt::format("&ramQuotaMB={}", bucket.ram_quota_mb));
-    encoded.body.append(fmt::format("&replicaNumber={}", bucket.num_replicas));
-    if (bucket.max_expiry > 0) {
-        encoded.body.append(fmt::format("&maxTTL={}", bucket.max_expiry));
+
+    if (bucket.ram_quota_mb > 0) {
+        encoded.body.append(fmt::format("&ramQuotaMB={}", bucket.ram_quota_mb));
+    }
+    if (bucket.num_replicas.has_value()) {
+        encoded.body.append(fmt::format("&replicaNumber={}", bucket.num_replicas.value()));
+    }
+
+    if (bucket.max_expiry.has_value()) {
+        encoded.body.append(fmt::format("&maxTTL={}", bucket.max_expiry.value()));
+    }
+    if (bucket.history_retention_collection_default.has_value()) {
+        encoded.body.append(
+          fmt::format("&historyRetentionCollectionDefault={}", bucket.history_retention_collection_default.value() ? "true" : "false"));
     }
-    encoded.body.append(fmt::format("&replicaIndex={}", bucket.replica_indexes ? "1" : "0"));
-    encoded.body.append(fmt::format("&flushEnabled={}", bucket.flush_enabled ? "1" : "0"));
+    if (bucket.history_retention_bytes.has_value()) {
+        encoded.body.append(fmt::format("&historyRetentionBytes={}", bucket.history_retention_bytes.value()));
+    }
+    if (bucket.history_retention_duration.has_value()) {
+        encoded.body.append(fmt::format("&historyRetentionSeconds={}", bucket.history_retention_duration.value()));
+    }
+    if (bucket.replica_indexes.has_value()) {
+        encoded.body.append(fmt::format("&replicaIndex={}", bucket.replica_indexes.value() ? "1" : "0"));
+    }
+    if (bucket.flush_enabled.has_value()) {
+        encoded.body.append(fmt::format("&flushEnabled={}", bucket.flush_enabled.value() ? "1" : "0"));
+    }
+
     switch (bucket.eviction_policy) {
         case couchbase::core::management::cluster::bucket_eviction_policy::full:
             encoded.body.append("&evictionPolicy=fullEviction");

data/ext/couchbase/core/operations/management/collection_create.cxx

@@ -35,6 +35,9 @@ collection_create_request::encode_to(encoded_request_type& encoded, http_context
     if (max_expiry > 0) {
         encoded.body.append(fmt::format("&maxTTL={}", max_expiry));
     }
+    if (history.has_value()) {
+        encoded.body.append(fmt::format("&history={}", history.value()));
+    }
     return {};
 }
 
@@ -48,7 +51,8 @@ collection_create_request::make_response(error_context::http&& ctx, const encode
                 std::regex collection_exists("Collection with name .+ already exists");
                 if (std::regex_search(encoded.body.data(), collection_exists)) {
                     response.ctx.ec = errc::management::collection_exists;
-                } else if (encoded.body.data().find("Not allowed on this version of cluster") != std::string::npos) {
+                } else if (encoded.body.data().find("Not allowed on this version of cluster") != std::string::npos ||
+                           encoded.body.data().find("Bucket must have storage_mode=magma") != std::string::npos) {
                     response.ctx.ec = errc::common::feature_not_available;
                 } else {
                     response.ctx.ec = errc::common::invalid_argument;

data/ext/couchbase/core/operations/management/collection_create.hxx

@@ -42,6 +42,7 @@ struct collection_create_request {
     std::string scope_name;
     std::string collection_name;
     std::uint32_t max_expiry{ 0 };
+    std::optional<bool> history{};
 
     std::optional<std::string> client_context_id{};
     std::optional<std::chrono::milliseconds> timeout{};

data/ext/couchbase/core/operations/management/collection_update.cxx

@@ -0,0 +1,87 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2020-2021 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "collection_update.hxx"
+
+#include "core/utils/json.hxx"
+#include "core/utils/url_codec.hxx"
+#include "error_utils.hxx"
+
+#include <regex>
+
+namespace couchbase::core::operations::management
+{
+std::error_code
+collection_update_request::encode_to(encoded_request_type& encoded, http_context& /* context */) const
+{
+    encoded.method = "PATCH";
+    encoded.path = fmt::format("/pools/default/buckets/{}/scopes/{}/collections/{}", bucket_name, scope_name, collection_name);
+    encoded.headers["content-type"] = "application/x-www-form-urlencoded";
+    std::map<std::string, std::string> values{};
+    if (max_expiry.has_value()) {
+        values["maxTTL"] = std::to_string(max_expiry.value());
+    }
+    if (history.has_value()) {
+        values["history"] = history.value() ? "true" : "false";
+    }
+    encoded.body = utils::string_codec::v2::form_encode(values);
+    return {};
+}
+
+collection_update_response
+collection_update_request::make_response(error_context::http&& ctx, const encoded_response_type& encoded) const
+{
+    collection_update_response response{ std::move(ctx) };
+    if (!response.ctx.ec) {
+        switch (encoded.status_code) {
+            case 400: {
+                if (encoded.body.data().find("Not allowed on this version of cluster") != std::string::npos ||
+                    encoded.body.data().find("Bucket must have storage_mode=magma") != std::string::npos) {
+                    response.ctx.ec = errc::common::feature_not_available;
+                } else {
+                    response.ctx.ec = errc::common::invalid_argument;
+                }
+            } break;
+            case 404: {
+                std::regex scope_not_found("Scope with name .+ is not found");
+                std::regex collection_not_found("Collection with name .+ is not found");
+                if (std::regex_search(encoded.body.data(), collection_not_found)) {
+                    response.ctx.ec = errc::common::collection_not_found;
+                } else if (std::regex_search(encoded.body.data(), scope_not_found)) {
+                    response.ctx.ec = errc::common::scope_not_found;
+                } else {
+                    response.ctx.ec = errc::common::bucket_not_found;
+                }
+            } break;
+            case 200: {
+                tao::json::value payload{};
+                try {
+                    payload = utils::json::parse(encoded.body.data());
+                } catch (const tao::pegtl::parse_error&) {
+                    response.ctx.ec = errc::common::parsing_failure;
+                    return response;
+                }
+                response.uid = std::stoull(payload.at("uid").get_string(), nullptr, 16);
+            } break;
+            default:
+                response.ctx.ec = extract_common_error_code(encoded.status_code, encoded.body.data());
+                break;
+        }
+    }
+    return response;
+}
+} // namespace couchbase::core::operations::management

data/ext/couchbase/core/operations/management/collection_update.hxx

@@ -0,0 +1,54 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2020-2021 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include "core/error_context/http.hxx"
+#include "core/io/http_context.hxx"
+#include "core/io/http_message.hxx"
+#include "core/platform/uuid.h"
+#include "core/timeout_defaults.hxx"
+
+namespace couchbase::core::operations::management
+{
+struct collection_update_response {
+    error_context::http ctx;
+    std::uint64_t uid{ 0 };
+};
+
+struct collection_update_request {
+    using response_type = collection_update_response;
+    using encoded_request_type = io::http_request;
+    using encoded_response_type = io::http_response;
+    using error_context_type = error_context::http;
+
+    static const inline service_type type = service_type::management;
+
+    std::string bucket_name;
+    std::string scope_name;
+    std::string collection_name;
+    std::optional<std::uint32_t> max_expiry{};
+    std::optional<bool> history{};
+
+    std::optional<std::string> client_context_id{};
+    std::optional<std::chrono::milliseconds> timeout{};
+
+    [[nodiscard]] std::error_code encode_to(encoded_request_type& encoded, http_context& context) const;
+
+    [[nodiscard]] collection_update_response make_response(error_context::http&& ctx, const encoded_response_type& encoded) const;
+};
+} // namespace couchbase::core::operations::management

data/ext/couchbase/core/operations/management/collections.hxx

@@ -19,6 +19,7 @@
 
 #include "collection_create.hxx"
 #include "collection_drop.hxx"
+#include "collection_update.hxx"
 #include "collections_manifest_get.hxx"
 #include "scope_create.hxx"
 #include "scope_drop.hxx"

data/ext/couchbase/core/operations.hxx

@@ -30,6 +30,8 @@
 #include "core/operations/document_increment.hxx"
 #include "core/operations/document_insert.hxx"
 #include "core/operations/document_lookup_in.hxx"
+#include "core/operations/document_lookup_in_all_replicas.hxx"
+#include "core/operations/document_lookup_in_any_replica.hxx"
 #include "core/operations/document_mutate_in.hxx"
 #include "core/operations/document_prepend.hxx"
 #include "core/operations/document_query.hxx"