couchbase 4.2.5 → 4.2.6-dev

package/deps/couchbase-cxx-client/bin/load-sample-buckets

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+#  Copyright 2020-2021 Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require "timeout"
+require "set"
+
+require_relative "api"
+
+options = {
+  host: ENV.fetch("CB_HOST", ARGV[0] || "127.0.0.1"),
+  strict_encryption: ENV.fetch("CB_STRICT_ENCRYPTION", ARGV[1]).to_b,
+  username: ENV.fetch("CB_USERNAME", ARGV[2] || "Administrator"),
+  password: ENV.fetch("CB_PASSWORD", ARGV[3] || "password"),
+  verbose: ENV.fetch("CB_VERBOSE", true).to_b,
+  bucket: "travel-sample",
+  sample_buckets: Set.new,
+}
+options[:sample_buckets] << "beer-sample" if ENV.fetch("CB_BEER_SAMPLE", false).to_b
+options[:sample_buckets] << "travel-sample" if ENV.fetch("CB_TRAVEL_SAMPLE", false).to_b
+options[:sample_buckets] |= ARGV[4..-1]
+return if options[:sample_buckets].empty?
+
+p options: options
+
+management_api =
+  begin
+    API.new(options.merge(port: options[:strict_encryption] ? 18091 : 8091))
+  rescue => ex
+    puts "#{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+
+expected_counts = {
+  "travel-sample" => 30_000,
+  "beer-sample" => 7_000,
+}
+
+options[:sample_buckets].each do |bucket|
+  ensure_sample_bucket_loaded(management_api, expected_counts[bucket], options.merge(bucket: bucket))
+end

package/deps/couchbase-cxx-client/core/cluster.hxx

@@ -79,6 +79,7 @@ class cluster : public std::enable_shared_from_this<cluster>
         }
 
         origin_ = std::move(origin);
+        CB_LOG_DEBUG(R"(open cluster, id: "{}", core version: "{}", {})", id_, couchbase::core::meta::sdk_semver(), origin_.to_json());
         // ignore the enable_tracing flag if a tracer was passed in
         if (nullptr != origin_.options().tracer) {
             tracer_ = origin_.options().tracer;
@@ -111,10 +112,7 @@ class cluster : public std::enable_shared_from_this<cluster>
                   return self->dns_srv_tracker_->get_srv_nodes(
                     [self, hostname = std::move(hostname), handler = std::forward<Handler>(handler)](origin::node_list nodes,
                                                                                                      std::error_code ec) mutable {
-                        if (ec) {
-                            return self->close([ec, handler = std::forward<Handler>(handler)]() mutable { handler(ec); });
-                        }
-                        if (!nodes.empty()) {
+                        if (!ec && !nodes.empty()) {
                             self->origin_.set_nodes(std::move(nodes));
                             CB_LOG_INFO("replace list of bootstrap nodes with addresses from DNS SRV of \"{}\": [{}]",
                                         hostname,
@@ -339,8 +337,8 @@ class cluster : public std::enable_shared_from_this<cluster>
     void do_open(Handler&& handler)
     {
         // Warn users if they attempt to use Capella without TLS being enabled.
+        bool has_capella_host = false;
         {
-            bool has_capella_host = false;
             bool has_non_capella_host = false;
             static std::string suffix = "cloud.couchbase.com";
             for (const auto& node : origin_.get_hostnames()) {
@@ -359,6 +357,7 @@ class cluster : public std::enable_shared_from_this<cluster>
 
             if (origin_.options().enable_tls                   /* TLS is enabled */
                 && origin_.options().trust_certificate.empty() /* No CA certificate (or other SDK-specific trust source) is specified */
+                && origin_.options().trust_certificate_value.empty()     /* and certificate value has not been specified */
                 && origin_.options().tls_verify != tls_verify_mode::none /* The user did not disable all TLS verification */
                 && has_non_capella_host /* The connection string has a hostname that does NOT end in ".cloud.couchbase.com" */) {
                 CB_LOG_WARNING("[{}] When TLS is enabled, the cluster options must specify certificate(s) to trust or ensure that they are "
@@ -368,7 +367,17 @@ class cluster : public std::enable_shared_from_this<cluster>
         }
 
         if (origin_.options().enable_tls) {
-            tls_.set_options(asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv2 | asio::ssl::context::no_sslv3);
+            long tls_options = asio::ssl::context::default_workarounds | // various bug workarounds that should be rather harmless
+                               asio::ssl::context::no_sslv2 |            // published: 1995, deprecated: 2011
+                               asio::ssl::context::no_sslv3;             // published: 1996, deprecated: 2015
+            if (origin_.options().tls_disable_deprecated_protocols) {
+                tls_options |= asio::ssl::context::no_tlsv1 |  // published: 1999, deprecated: 2021
+                               asio::ssl::context::no_tlsv1_1; // published: 2006, deprecated: 2021
+            }
+            if (origin_.options().tls_disable_v1_2 || has_capella_host) {
+                tls_options |= asio::ssl::context::no_tlsv1_2; // published: 2008, still in use
+            }
+            tls_.set_options(tls_options);
             switch (origin_.options().tls_verify) {
                 case tls_verify_mode::none:
                     tls_.set_verify_mode(asio::ssl::verify_none);
@@ -378,7 +387,8 @@ class cluster : public std::enable_shared_from_this<cluster>
                     tls_.set_verify_mode(asio::ssl::verify_peer);
                     break;
             }
-            if (origin_.options().trust_certificate.empty()) { // trust certificate is not explicitly specified
+            if (origin_.options().trust_certificate.empty() &&
+                origin_.options().trust_certificate_value.empty()) { // trust certificate is not explicitly specified
                 CB_LOG_DEBUG(R"([{}]: use default CA for TLS verify)", id_);
                 std::error_code ec{};
 
@@ -414,11 +424,22 @@ class cluster : public std::enable_shared_from_this<cluster>
                 std::error_code ec{};
                 // load only the explicit certificate
                 // system and default capella certificates are not loaded
-                CB_LOG_DEBUG(R"([{}]: use TLS verify file: "{}")", id_, origin_.options().trust_certificate);
-                tls_.load_verify_file(origin_.options().trust_certificate, ec);
-                if (ec) {
-                    CB_LOG_ERROR("[{}]: unable to load verify file \"{}\": {}", id_, origin_.options().trust_certificate, ec.message());
-                    return close([ec, handler = std::forward<Handler>(handler)]() mutable { return handler(ec); });
+                if (!origin_.options().trust_certificate_value.empty()) {
+                    CB_LOG_DEBUG(R"([{}]: use TLS certificate passed through via options object)", id_);
+                    tls_.add_certificate_authority(asio::const_buffer(origin_.options().trust_certificate_value.data(),
+                                                                      origin_.options().trust_certificate_value.size()),
+                                                   ec);
+                    if (ec) {
+                        CB_LOG_WARNING("[{}]: unable to load CA passed via options object: {}", id_, ec.message());
+                    }
+                }
+                if (!origin_.options().trust_certificate.empty()) {
+                    CB_LOG_DEBUG(R"([{}]: use TLS verify file: "{}")", id_, origin_.options().trust_certificate);
+                    tls_.load_verify_file(origin_.options().trust_certificate, ec);
+                    if (ec) {
+                        CB_LOG_ERROR("[{}]: unable to load verify file \"{}\": {}", id_, origin_.options().trust_certificate, ec.message());
+                        return close([ec, handler = std::forward<Handler>(handler)]() mutable { return handler(ec); });
+                    }
                 }
             }
 #ifdef COUCHBASE_CXX_CLIENT_TLS_KEY_LOG_FILE

package/deps/couchbase-cxx-client/core/cluster_options.hxx

@@ -50,7 +50,10 @@ struct cluster_options {
     std::chrono::milliseconds management_timeout = timeout_defaults::management_timeout;
 
     bool enable_tls{ false };
+    bool tls_disable_deprecated_protocols{ true };
+    bool tls_disable_v1_2{ false };
     std::string trust_certificate{};
+    std::string trust_certificate_value{};
     bool enable_mutation_tokens{ true };
     bool enable_tcp_keep_alive{ true };
     io::ip_protocol use_ip_protocol{ io::ip_protocol::any };

package/deps/couchbase-cxx-client/core/crud_component.cxx

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
 /*
- *   Copyright 2020-2021 Couchbase, Inc.
+ *   Copyright 2020-2023 Couchbase, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -39,6 +39,8 @@
 
 #include <tl/expected.hpp>
 
+#include <random>
+
 namespace couchbase::core
 {
 static std::pair<std::vector<std::byte>, std::error_code>
@@ -53,17 +55,40 @@ serialize_range_scan_create_options(const range_scan_create_options& options)
         body["collection"] = fmt::format("{:x}", options.collection_id);
     }
 
-    if (std::holds_alternative<range_scan>(options.scan_type)) {
-        const auto& range = std::get<range_scan>(options.scan_type);
+    if (std::holds_alternative<range_scan>(options.scan_type) || std::holds_alternative<prefix_scan>(options.scan_type)) {
+        const auto& range = (std::holds_alternative<range_scan>(options.scan_type))
+                              ? std::get<range_scan>(options.scan_type)
+                              : std::get<prefix_scan>(options.scan_type).to_range_scan();
+
+        const auto& from = range.from.value_or(scan_term{ "" });
+        const auto& to = range.to.value_or(scan_term{ "\xf4\x8f\xfb\xfb" });
+
         body["range"] = {
-            { range.start_.exclusive ? "excl_start" : "start", base64::encode(range.start_.id) },
-            { range.end_.exclusive ? "excl_end" : "end", base64::encode(range.end_.id) },
+            { from.exclusive ? "excl_start" : "start", base64::encode(from.term) },
+            { to.exclusive ? "excl_end" : "end", base64::encode(to.term) },
         };
     } else if (std::holds_alternative<sampling_scan>(options.scan_type)) {
         const auto& sampling = std::get<sampling_scan>(options.scan_type);
+
+        // The limit in sampling scan is required to be greater than 0
+        if (sampling.limit <= 0) {
+            return { {}, errc::common::invalid_argument };
+        }
+
+        std::uint64_t seed{};
+        if (sampling.seed.has_value()) {
+            seed = sampling.seed.value();
+        } else {
+            // Generate random uint64 as seed
+            std::random_device rd;
+            std::mt19937_64 gen(rd());
+            std::uniform_int_distribution<std::uint64_t> dis;
+            seed = dis(gen);
+        }
+
         body["sampling"] = {
             { "samples", sampling.limit },
-            { "seed", sampling.seed.value_or(0) },
+            { "seed", seed },
         };
     } else {
         return { {}, errc::common::invalid_argument };
@@ -75,7 +100,7 @@ serialize_range_scan_create_options(const range_scan_create_options& options)
             { "vb_uuid", std::to_string(snapshot.vbucket_uuid) },
             { "seqno", snapshot.sequence_number },
             { "timeout_ms",
-              (options.timeout == std::chrono::milliseconds::zero()) ? timeout_defaults::range_scan_timeout.count()
+              (options.timeout == std::chrono::milliseconds::zero()) ? timeout_defaults::key_value_scan_timeout.count()
                                                                      : options.timeout.count() },
         };
         if (snapshot.sequence_number_exists) {
@@ -98,7 +123,7 @@ parse_range_scan_keys(gsl::span<std::byte> data, range_scan_item_callback&& item
         if (remaining.size() < key_length) {
             return errc::network::protocol_error;
         }
-        item_callback(range_scan_item{ { remaining.begin(), remaining.begin() + static_cast<std::ptrdiff_t>(key_length) } });
+        item_callback(range_scan_item{ { reinterpret_cast<const char*>(remaining.data()), key_length } });
         if (remaining.size() == key_length) {
             return {};
         }
@@ -130,13 +155,13 @@ parse_range_scan_documents(gsl::span<std::byte> data, range_scan_item_callback&&
         body.datatype = data[24];
         data = gsl::make_span(data.data() + header_offset, data.size() - header_offset);
 
-        std::vector<std::byte> key{};
+        std::string key{};
         {
             auto [key_length, remaining] = utils::decode_unsigned_leb128<std::size_t>(data, core::utils::leb_128_no_throw{});
             if (remaining.size() < key_length) {
                 return errc::network::protocol_error;
             }
-            key = { remaining.begin(), remaining.begin() + static_cast<std::ptrdiff_t>(key_length) };
+            key = { reinterpret_cast<const char*>(remaining.data()), key_length };
             data = gsl::make_span(remaining.data() + key_length, remaining.size() - key_length);
         }
 
@@ -242,19 +267,10 @@ class crud_component_impl
               if (error) {
                   return cb({}, error);
               }
-              bool ids_only;
-              switch (response->extras_.size()) {
-                  case 4:
-                      ids_only = mcbp::big_endian::read_uint32(response->extras_, 0) == 0;
-                      break;
-
-                  case 0:
-                      ids_only = options.ids_only; // support servers before MB-54267. TODO: remove after server GA
-                      break;
-
-                  default:
-                      return cb({}, errc::network::protocol_error);
+              if (response->extras_.size() != 4) {
+                  return cb({}, errc::network::protocol_error);
               }
+              bool ids_only = mcbp::big_endian::read_uint32(response->extras_, 0) == 0;
 
               if (auto ec = parse_range_scan_data(response->value_, std::move(item_cb), ids_only); ec) {
                   return cb({}, ec);
@@ -276,6 +292,19 @@ class crud_component_impl
 
         req->persistent_ = true;
         req->vbucket_ = vbucket_id;
+
+        if (options.timeout != std::chrono::milliseconds::zero()) {
+            auto timer = std::make_shared<asio::steady_timer>(io_);
+            timer->expires_after(options.timeout);
+            timer->async_wait([req](auto error) {
+                if (error == asio::error::operation_aborted) {
+                    return;
+                }
+                req->cancel(couchbase::errc::common::unambiguous_timeout);
+            });
+            req->set_deadline(timer);
+        }
+
         mcbp::buffer_writer buf{ scan_uuid.size() + sizeof(std::uint32_t) * 3 };
         buf.write(scan_uuid);
         buf.write_uint32(options.batch_item_limit);

package/deps/couchbase-cxx-client/core/impl/build_deferred_query_indexes.cxx

@@ -20,44 +20,116 @@
 
 #include "core/cluster.hxx"
 #include "core/operations/management/query_index_build.hxx"
-#include "core/operations/management/query_index_get_all_deferred.hxx"
-#include "core/utils/json.hxx"
+#include "core/operations/management/query_index_build_deferred.hxx"
 
-namespace couchbase::core::impl
+namespace couchbase
 {
 template<typename Response>
 static manager_error_context
 build_context(Response& resp)
 {
-    return { resp.ctx.ec,
-             resp.ctx.last_dispatched_to,
-             resp.ctx.last_dispatched_from,
-             resp.ctx.retry_attempts,
-             std::move(resp.ctx.retry_reasons),
-             std::move(resp.ctx.client_context_id),
-             resp.ctx.http_status,
-             std::move(resp.ctx.http_body),
-             std::move(resp.ctx.path) };
+    return manager_error_context(internal_manager_error_context{ resp.ctx.ec,
+                                                                 resp.ctx.last_dispatched_to,
+                                                                 resp.ctx.last_dispatched_from,
+                                                                 resp.ctx.retry_attempts,
+                                                                 std::move(resp.ctx.retry_reasons),
+                                                                 std::move(resp.ctx.client_context_id),
+                                                                 resp.ctx.http_status,
+                                                                 std::move(resp.ctx.http_body),
+                                                                 std::move(resp.ctx.path) });
 }
+
+static core::operations::management::query_index_build_request
+build_build_index_request(std::string bucket_name,
+                          core::operations::management::query_index_get_all_deferred_response list_resp,
+                          const build_query_index_options::built& options)
+{
+    core::operations::management::query_index_build_request request{
+        std::move(bucket_name), {}, {}, {}, std::move(list_resp.index_names), {}, options.timeout
+    };
+    return request;
+}
+
+static core::operations::management::query_index_get_all_deferred_request
+build_get_all_request(std::string bucket_name, const build_query_index_options::built& options)
+{
+    core::operations::management::query_index_get_all_deferred_request request{ std::move(bucket_name), {}, {}, {}, {}, options.timeout };
+    return request;
+}
+
+static core::operations::management::query_index_build_request
+build_build_index_request(std::string bucket_name,
+                          std::string scope_name,
+                          std::string collection_name,
+                          core::operations::management::query_index_get_all_deferred_response list_resp,
+                          const build_query_index_options::built& options)
+{
+    core::operations::management::query_index_build_request request{ "",
+                                                                     "",
+                                                                     std::move(collection_name),
+                                                                     core::query_context{ std::move(bucket_name), std::move(scope_name) },
+                                                                     std::move(list_resp.index_names),
+                                                                     {},
+                                                                     options.timeout };
+    return request;
+}
+
+static core::operations::management::query_index_get_all_deferred_request
+build_get_all_request(std::string bucket_name,
+                      std::string scope_name,
+                      std::string collection_name,
+                      const build_query_index_options::built& options)
+{
+    core::operations::management::query_index_get_all_deferred_request request{
+        "", "", std::move(collection_name), core::query_context{ std::move(bucket_name), std::move(scope_name) }, {}, options.timeout
+    };
+    return request;
+}
+
 void
-initiate_build_deferred_indexes(std::shared_ptr<couchbase::core::cluster> core,
-                                std::string bucket_name,
-                                build_query_index_options::built options,
-                                query_context query_ctx,
-                                std::string collection_name,
-                                build_deferred_query_indexes_handler&& handler)
+query_index_manager::build_deferred_indexes(std::string bucket_name,
+                                            const couchbase::build_query_index_options& options,
+                                            couchbase::build_deferred_query_indexes_handler&& handler) const
+{
+
+    auto get_all_request = build_get_all_request(bucket_name, options.build());
+    core_->execute(std::move(get_all_request),
+                   [handler = std::move(handler), this, bucket_name, options](
+                     core::operations::management::query_index_get_all_deferred_response resp1) mutable {
+                       auto list_resp = std::move(resp1);
+                       if (list_resp.ctx.ec) {
+                           return handler(build_context(list_resp));
+                       }
+                       if (list_resp.index_names.empty()) {
+                           return handler(build_context(list_resp));
+                       }
+                       auto build_request = build_build_index_request(std::move(bucket_name), list_resp, options.build());
+                       core_->execute(
+                         std::move(build_request),
+                         [handler = std::move(handler)](core::operations::management::query_index_build_response resp2) mutable {
+                             auto build_resp = std::move(resp2);
+                             return handler(build_context(build_resp));
+                         });
+                   });
+}
+
+auto
+query_index_manager::build_deferred_indexes(std::string bucket_name, const couchbase::build_query_index_options& options) const
+  -> std::future<manager_error_context>
 {
-    core->execute(
-      operations::management::query_index_get_all_deferred_request{
-        bucket_name,
-        "",
-        collection_name,
-        query_ctx,
-        {},
-        options.timeout,
-      },
-      [core, bucket_name, collection_name, options = std::move(options), query_ctx, handler = std::move(handler)](
-        operations::management::query_index_get_all_deferred_response resp1) mutable {
+    auto barrier = std::make_shared<std::promise<manager_error_context>>();
+    build_deferred_indexes(std::move(bucket_name), options, [barrier](auto ctx) mutable { barrier->set_value(std::move(ctx)); });
+    return barrier->get_future();
+}
+
+void
+collection_query_index_manager::build_deferred_indexes(const build_query_index_options& options,
+                                                       build_deferred_query_indexes_handler&& handler) const
+{
+    auto get_all_request = build_get_all_request(bucket_name_, scope_name_, collection_name_, options.build());
+    core_->execute(
+      std::move(get_all_request),
+      [handler = std::move(handler), this, options](core::operations::management::query_index_get_all_deferred_response resp1) mutable {
           auto list_resp = std::move(resp1);
           if (list_resp.ctx.ec) {
               return handler(build_context(list_resp));
@@ -65,28 +137,21 @@ initiate_build_deferred_indexes(std::shared_ptr<couchbase::core::cluster> core,
           if (list_resp.index_names.empty()) {
               return handler(build_context(list_resp));
           }
-          core->execute(
-            operations::management::query_index_build_request{
-              std::move(bucket_name),
-              "",
-              collection_name,
-              query_ctx,
-              std::move(list_resp.index_names),
-              {},
-              options.timeout,
-            },
-            [handler = std::move(handler)](operations::management::query_index_build_response resp2) {
-                auto build_resp = std::move(resp2);
-                return handler(build_context(build_resp));
-            });
+          auto build_request = build_build_index_request(bucket_name_, scope_name_, collection_name_, list_resp, options.build());
+          core_->execute(std::move(build_request),
+                         [handler = std::move(handler)](core::operations::management::query_index_build_response resp2) mutable {
+                             auto build_resp = std::move(resp2);
+                             return handler(build_context(build_resp));
+                         });
       });
 }
-void
-initiate_build_deferred_indexes(std::shared_ptr<couchbase::core::cluster> core,
-                                std::string bucket_name,
-                                build_query_index_options::built options,
-                                build_deferred_query_indexes_handler&& handler)
+
+auto
+collection_query_index_manager::build_deferred_indexes(const couchbase::build_query_index_options& options) const
+  -> std::future<manager_error_context>
 {
-    return initiate_build_deferred_indexes(core, std::move(bucket_name), options, {}, "", std::move(handler));
+    auto barrier = std::make_shared<std::promise<manager_error_context>>();
+    build_deferred_indexes(options, [barrier](auto ctx) mutable { barrier->set_value(std::move(ctx)); });
+    return barrier->get_future();
 }
-} // namespace couchbase::core::impl
+} // namespace couchbase

package/deps/couchbase-cxx-client/core/impl/cluster.cxx

@@ -82,6 +82,9 @@ options_to_origin(const std::string& connection_string, const couchbase::cluster
         if (opts.security.trust_certificate.has_value()) {
             user_options.trust_certificate = opts.security.trust_certificate.value();
         }
+        if (opts.security.trust_certificate_value.has_value()) {
+            user_options.trust_certificate_value = opts.security.trust_certificate_value.value();
+        }
         switch (opts.security.tls_verify) {
             case couchbase::tls_verify_mode::none:
                 user_options.tls_verify = core::tls_verify_mode::none;
@@ -91,6 +94,8 @@ options_to_origin(const std::string& connection_string, const couchbase::cluster
                 break;
         }
         user_options.disable_mozilla_ca_certificates = opts.security.disable_mozilla_ca_certificates;
+        user_options.tls_disable_deprecated_protocols = opts.security.disable_deprecated_protocols;
+        user_options.tls_disable_v1_2 = opts.security.disable_tls_v1_2;
     }
 
     if (opts.dns.nameserver) {
@@ -103,6 +108,7 @@ options_to_origin(const std::string& connection_string, const couchbase::cluster
     user_options.enable_mutation_tokens = opts.behavior.enable_mutation_tokens;
     user_options.enable_unordered_execution = opts.behavior.enable_unordered_execution;
     user_options.user_agent_extra = opts.behavior.user_agent_extra;
+    user_options.network = opts.behavior.network;
 
     user_options.enable_tcp_keep_alive = opts.network.enable_tcp_keep_alive;
     user_options.tcp_keep_alive_interval = opts.network.tcp_keep_alive_interval;