couchbase 4.2.5-dev.3 → 4.2.6-dev

package/deps/couchbase-cxx-client/bin/init-cluster

@@ -17,13 +17,8 @@
 require "set"
 require "timeout"
 require "optparse"
-require "openssl"
 
-class Object
-  def to_b
-    ![nil, false, 0, "", "0", "f", "F", "false", "FALSE", "off", "OFF", "no", "NO"].include?(self)
-  end
-end
+require_relative "api"
 
 options = {
   verbose: ENV.fetch("CB_VERBOSE", true).to_b,
@@ -49,106 +44,6 @@ options[:port] = ENV.fetch("CB_PORT", default_port).to_i
 options[:sample_buckets] << "beer-sample" if ENV.fetch("CB_BEER_SAMPLE", false).to_b
 options[:sample_buckets] << "travel-sample" if ENV.fetch("CB_TRAVEL_SAMPLE", false).to_b
 
-require "net/http"
-require "json"
-
-class API
-  def initialize(options)
-    @options = options
-    connect
-  end
-
-  def connect
-    @client = Net::HTTP.start(@options[:host], @options[:port],
-                              use_ssl: @options[:strict_encryption],
-                              verify_mode: OpenSSL::SSL::VERIFY_NONE)
-  end
-
-  def url(path)
-    "#{@client.use_ssl? ? 'https' : 'http'}://#{@options[:username]}@#{@options[:host]}:#{@options[:port]}#{path}"
-  end
-
-  def decode_response(response)
-    payload =
-      if /application\/json/.match?(response['content-type'])
-        JSON.parse(response.body)
-      else
-        response.body
-      end
-    p payload if @options[:verbose]
-    payload
-  end
-
-  def setup_request(request)
-    request.basic_auth(@options[:username], @options[:password])
-    request['accept'] = "application/json"
-  end
-
-  def get(path)
-    puts "# GET #{url(path)}"
-    req = Net::HTTP::Get.new(path)
-    setup_request(req)
-    res = @client.request(req)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def post_form(path, fields = {})
-    puts "# POST #{url(path)} #{fields}"
-    req = Net::HTTP::Post.new(path)
-    setup_request(req)
-    req.form_data = fields
-    res = @client.request(req)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def post_json(path, object)
-    data = JSON.generate(object)
-    puts "# POST #{url(path)} #{data}"
-    req = Net::HTTP::Post.new(path)
-    req['content-type'] = "application/json"
-    setup_request(req)
-    res = @client.request(req, data)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def put_json(path, object)
-    data = JSON.generate(object)
-    puts "# PUT #{url(path)} #{data}"
-    req = Net::HTTP::Put.new(path)
-    req['content-type'] = "application/json"
-    setup_request(req)
-    res = @client.request(req, data)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-end
 
 api = 
   begin
@@ -180,7 +75,10 @@ api.post_form("/settings/web",
               password: options[:password],
               username: options[:username],
               port: "SAME")
-api.post_form("/settings/indexes", storageMode: "plasma")
+res = api.post_form("/settings/indexes", storageMode: "plasma")
+if res["errors"]
+  res = api.post_form("/settings/indexes", storageMode: "forestdb")
+end
 
 if options[:cluster_run_nodes] > 1
   known_nodes = []
@@ -235,50 +133,30 @@ if options[:sec_bucket].to_b
               name: options[:sec_bucket])
 end
 
-api.post_json("/sampleBuckets/install", options[:sample_buckets].to_a) if options[:sample_buckets].any?
-
 api.post_form("/settings/developerPreview", enabled: true) if options[:enable_developer_preview]
 
-def service_port_for_bucket(api, service, bucket, options)
-  port_key = options[:strict_encryption] ? "#{service}SSL" : service
-  port = 0
-  while port.zero?
-    config = api.get("/pools/default/b/#{bucket}")
-    port = config["nodesExt"].find{|n| n["services"].key?(port_key)}["services"][port_key].to_i rescue 0
-    sleep 1
-  end
-  port
-end
+service_address = service_address_for_bucket(api, "n1ql", options[:bucket], options)
+puts query_service: service_address
 
-query_api = API.new(options.merge(port: service_port_for_bucket(api, "n1ql", options[:bucket], options)))
+query_api = API.new(options.merge(service_address))
 query_api.post_form("/query/service", statement: "CREATE PRIMARY INDEX ON `#{options[:bucket]}` USING GSI")
+
+expected_counts = {
+  "travel-sample" => 30_000,
+  "beer-sample" => 7_000,
+}
 options[:sample_buckets].each do |bucket|
-  service_port_for_bucket(api, "n1ql", bucket, options)
-  query_api.post_form("/query/service", statement: "CREATE PRIMARY INDEX ON `#{bucket}` USING GSI")
+  ensure_sample_bucket_loaded(api, expected_counts[bucket], options.merge(bucket: bucket))
 end
 
-puts service_port_for_bucket(api, "fts", options[:bucket], options)
-
 if options[:sample_buckets].include?("travel-sample") && services.include?("fts")
-  search_api = API.new(options.merge(port: service_port_for_bucket(api, "fts", options[:bucket], options)))
+  service_address = service_address_for_bucket(api, "fts", options[:bucket], options)
+  puts search_service: service_address
 
   has_v6_nodes = 
     api.get("/pools/default")["nodes"]
     .any? { |node| node["version"] =~ /^6\./ && node["services"].include?("fts") }
-
   index_definition_path = File.join(__dir__, "travel-sample-index#{"-v6" if has_v6_nodes}.json")
-  puts "using index definition: #{File.basename(index_definition_path)}"
-  index_definition = JSON.load(File.read(index_definition_path))
-  search_api.put_json("/api/index/travel-sample-index", index_definition)
-
-  expected_number_of_the_documents = 1000
-  indexed_documents = 0
-  Timeout.timeout(10 * 60) do # give it 10 minutes to index
-    while indexed_documents < expected_number_of_the_documents
-      resp = search_api.get("/api/index/travel-sample-index/count")
-      indexed_documents = resp['count'].to_i
-      sleep 1
-    end
-  end
+  ensure_search_index_created(index_definition_path, options.merge(service_address))
 end
 

package/deps/couchbase-cxx-client/bin/load-sample-buckets

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+#  Copyright 2020-2021 Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require "timeout"
+require "set"
+
+require_relative "api"
+
+options = {
+  host: ENV.fetch("CB_HOST", ARGV[0] || "127.0.0.1"),
+  strict_encryption: ENV.fetch("CB_STRICT_ENCRYPTION", ARGV[1]).to_b,
+  username: ENV.fetch("CB_USERNAME", ARGV[2] || "Administrator"),
+  password: ENV.fetch("CB_PASSWORD", ARGV[3] || "password"),
+  verbose: ENV.fetch("CB_VERBOSE", true).to_b,
+  bucket: "travel-sample",
+  sample_buckets: Set.new,
+}
+options[:sample_buckets] << "beer-sample" if ENV.fetch("CB_BEER_SAMPLE", false).to_b
+options[:sample_buckets] << "travel-sample" if ENV.fetch("CB_TRAVEL_SAMPLE", false).to_b
+options[:sample_buckets] |= ARGV[4..-1]
+return if options[:sample_buckets].empty?
+
+p options: options
+
+management_api =
+  begin
+    API.new(options.merge(port: options[:strict_encryption] ? 18091 : 8091))
+  rescue => ex
+    puts "#{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+
+expected_counts = {
+  "travel-sample" => 30_000,
+  "beer-sample" => 7_000,
+}
+
+options[:sample_buckets].each do |bucket|
+  ensure_sample_bucket_loaded(management_api, expected_counts[bucket], options.merge(bucket: bucket))
+end

package/deps/couchbase-cxx-client/core/cluster.hxx

@@ -79,6 +79,7 @@ class cluster : public std::enable_shared_from_this<cluster>
         }
 
         origin_ = std::move(origin);
+        CB_LOG_DEBUG(R"(open cluster, id: "{}", core version: "{}", {})", id_, couchbase::core::meta::sdk_semver(), origin_.to_json());
         // ignore the enable_tracing flag if a tracer was passed in
         if (nullptr != origin_.options().tracer) {
             tracer_ = origin_.options().tracer;
@@ -111,10 +112,7 @@ class cluster : public std::enable_shared_from_this<cluster>
                   return self->dns_srv_tracker_->get_srv_nodes(
                     [self, hostname = std::move(hostname), handler = std::forward<Handler>(handler)](origin::node_list nodes,
                                                                                                      std::error_code ec) mutable {
-                        if (ec) {
-                            return self->close([ec, handler = std::forward<Handler>(handler)]() mutable { handler(ec); });
-                        }
-                        if (!nodes.empty()) {
+                        if (!ec && !nodes.empty()) {
                             self->origin_.set_nodes(std::move(nodes));
                             CB_LOG_INFO("replace list of bootstrap nodes with addresses from DNS SRV of \"{}\": [{}]",
                                         hostname,
@@ -339,8 +337,8 @@ class cluster : public std::enable_shared_from_this<cluster>
     void do_open(Handler&& handler)
     {
         // Warn users if they attempt to use Capella without TLS being enabled.
+        bool has_capella_host = false;
         {
-            bool has_capella_host = false;
             bool has_non_capella_host = false;
             static std::string suffix = "cloud.couchbase.com";
             for (const auto& node : origin_.get_hostnames()) {
@@ -359,6 +357,7 @@ class cluster : public std::enable_shared_from_this<cluster>
 
             if (origin_.options().enable_tls                   /* TLS is enabled */
                 && origin_.options().trust_certificate.empty() /* No CA certificate (or other SDK-specific trust source) is specified */
+                && origin_.options().trust_certificate_value.empty()     /* and certificate value has not been specified */
                 && origin_.options().tls_verify != tls_verify_mode::none /* The user did not disable all TLS verification */
                 && has_non_capella_host /* The connection string has a hostname that does NOT end in ".cloud.couchbase.com" */) {
                 CB_LOG_WARNING("[{}] When TLS is enabled, the cluster options must specify certificate(s) to trust or ensure that they are "
@@ -368,7 +367,17 @@ class cluster : public std::enable_shared_from_this<cluster>
         }
 
         if (origin_.options().enable_tls) {
-            tls_.set_options(asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv2 | asio::ssl::context::no_sslv3);
+            long tls_options = asio::ssl::context::default_workarounds | // various bug workarounds that should be rather harmless
+                               asio::ssl::context::no_sslv2 |            // published: 1995, deprecated: 2011
+                               asio::ssl::context::no_sslv3;             // published: 1996, deprecated: 2015
+            if (origin_.options().tls_disable_deprecated_protocols) {
+                tls_options |= asio::ssl::context::no_tlsv1 |  // published: 1999, deprecated: 2021
+                               asio::ssl::context::no_tlsv1_1; // published: 2006, deprecated: 2021
+            }
+            if (origin_.options().tls_disable_v1_2 || has_capella_host) {
+                tls_options |= asio::ssl::context::no_tlsv1_2; // published: 2008, still in use
+            }
+            tls_.set_options(tls_options);
             switch (origin_.options().tls_verify) {
                 case tls_verify_mode::none:
                     tls_.set_verify_mode(asio::ssl::verify_none);
@@ -378,7 +387,8 @@ class cluster : public std::enable_shared_from_this<cluster>
                     tls_.set_verify_mode(asio::ssl::verify_peer);
                     break;
             }
-            if (origin_.options().trust_certificate.empty()) { // trust certificate is not explicitly specified
+            if (origin_.options().trust_certificate.empty() &&
+                origin_.options().trust_certificate_value.empty()) { // trust certificate is not explicitly specified
                 CB_LOG_DEBUG(R"([{}]: use default CA for TLS verify)", id_);
                 std::error_code ec{};
 
@@ -414,11 +424,22 @@ class cluster : public std::enable_shared_from_this<cluster>
                 std::error_code ec{};
                 // load only the explicit certificate
                 // system and default capella certificates are not loaded
-                CB_LOG_DEBUG(R"([{}]: use TLS verify file: "{}")", id_, origin_.options().trust_certificate);
-                tls_.load_verify_file(origin_.options().trust_certificate, ec);
-                if (ec) {
-                    CB_LOG_ERROR("[{}]: unable to load verify file \"{}\": {}", id_, origin_.options().trust_certificate, ec.message());
-                    return close([ec, handler = std::forward<Handler>(handler)]() mutable { return handler(ec); });
+                if (!origin_.options().trust_certificate_value.empty()) {
+                    CB_LOG_DEBUG(R"([{}]: use TLS certificate passed through via options object)", id_);
+                    tls_.add_certificate_authority(asio::const_buffer(origin_.options().trust_certificate_value.data(),
+                                                                      origin_.options().trust_certificate_value.size()),
+                                                   ec);
+                    if (ec) {
+                        CB_LOG_WARNING("[{}]: unable to load CA passed via options object: {}", id_, ec.message());
+                    }
+                }
+                if (!origin_.options().trust_certificate.empty()) {
+                    CB_LOG_DEBUG(R"([{}]: use TLS verify file: "{}")", id_, origin_.options().trust_certificate);
+                    tls_.load_verify_file(origin_.options().trust_certificate, ec);
+                    if (ec) {
+                        CB_LOG_ERROR("[{}]: unable to load verify file \"{}\": {}", id_, origin_.options().trust_certificate, ec.message());
+                        return close([ec, handler = std::forward<Handler>(handler)]() mutable { return handler(ec); });
+                    }
                 }
             }
 #ifdef COUCHBASE_CXX_CLIENT_TLS_KEY_LOG_FILE

package/deps/couchbase-cxx-client/core/cluster_options.hxx

@@ -50,7 +50,10 @@ struct cluster_options {
     std::chrono::milliseconds management_timeout = timeout_defaults::management_timeout;
 
     bool enable_tls{ false };
+    bool tls_disable_deprecated_protocols{ true };
+    bool tls_disable_v1_2{ false };
     std::string trust_certificate{};
+    std::string trust_certificate_value{};
     bool enable_mutation_tokens{ true };
     bool enable_tcp_keep_alive{ true };
     io::ip_protocol use_ip_protocol{ io::ip_protocol::any };

package/deps/couchbase-cxx-client/core/crud_component.cxx

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
 /*
- *   Copyright 2020-2021 Couchbase, Inc.
+ *   Copyright 2020-2023 Couchbase, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -39,6 +39,8 @@
 
 #include <tl/expected.hpp>
 
+#include <random>
+
 namespace couchbase::core
 {
 static std::pair<std::vector<std::byte>, std::error_code>
@@ -53,17 +55,40 @@ serialize_range_scan_create_options(const range_scan_create_options& options)
         body["collection"] = fmt::format("{:x}", options.collection_id);
     }
 
-    if (std::holds_alternative<range_scan>(options.scan_type)) {
-        const auto& range = std::get<range_scan>(options.scan_type);
+    if (std::holds_alternative<range_scan>(options.scan_type) || std::holds_alternative<prefix_scan>(options.scan_type)) {
+        const auto& range = (std::holds_alternative<range_scan>(options.scan_type))
+                              ? std::get<range_scan>(options.scan_type)
+                              : std::get<prefix_scan>(options.scan_type).to_range_scan();
+
+        const auto& from = range.from.value_or(scan_term{ "" });
+        const auto& to = range.to.value_or(scan_term{ "\xf4\x8f\xfb\xfb" });
+
         body["range"] = {
-            { range.start_.exclusive ? "excl_start" : "start", base64::encode(range.start_.id) },
-            { range.end_.exclusive ? "excl_end" : "end", base64::encode(range.end_.id) },
+            { from.exclusive ? "excl_start" : "start", base64::encode(from.term) },
+            { to.exclusive ? "excl_end" : "end", base64::encode(to.term) },
         };
     } else if (std::holds_alternative<sampling_scan>(options.scan_type)) {
         const auto& sampling = std::get<sampling_scan>(options.scan_type);
+
+        // The limit in sampling scan is required to be greater than 0
+        if (sampling.limit <= 0) {
+            return { {}, errc::common::invalid_argument };
+        }
+
+        std::uint64_t seed{};
+        if (sampling.seed.has_value()) {
+            seed = sampling.seed.value();
+        } else {
+            // Generate random uint64 as seed
+            std::random_device rd;
+            std::mt19937_64 gen(rd());
+            std::uniform_int_distribution<std::uint64_t> dis;
+            seed = dis(gen);
+        }
+
         body["sampling"] = {
             { "samples", sampling.limit },
-            { "seed", sampling.seed.value_or(0) },
+            { "seed", seed },
         };
     } else {
         return { {}, errc::common::invalid_argument };
@@ -75,7 +100,7 @@ serialize_range_scan_create_options(const range_scan_create_options& options)
             { "vb_uuid", std::to_string(snapshot.vbucket_uuid) },
             { "seqno", snapshot.sequence_number },
             { "timeout_ms",
-              (options.timeout == std::chrono::milliseconds::zero()) ? timeout_defaults::range_scan_timeout.count()
+              (options.timeout == std::chrono::milliseconds::zero()) ? timeout_defaults::key_value_scan_timeout.count()
                                                                      : options.timeout.count() },
         };
         if (snapshot.sequence_number_exists) {
@@ -98,7 +123,7 @@ parse_range_scan_keys(gsl::span<std::byte> data, range_scan_item_callback&& item
         if (remaining.size() < key_length) {
             return errc::network::protocol_error;
         }
-        item_callback(range_scan_item{ { remaining.begin(), remaining.begin() + static_cast<std::ptrdiff_t>(key_length) } });
+        item_callback(range_scan_item{ { reinterpret_cast<const char*>(remaining.data()), key_length } });
         if (remaining.size() == key_length) {
             return {};
         }
@@ -130,13 +155,13 @@ parse_range_scan_documents(gsl::span<std::byte> data, range_scan_item_callback&&
         body.datatype = data[24];
         data = gsl::make_span(data.data() + header_offset, data.size() - header_offset);
 
-        std::vector<std::byte> key{};
+        std::string key{};
         {
             auto [key_length, remaining] = utils::decode_unsigned_leb128<std::size_t>(data, core::utils::leb_128_no_throw{});
             if (remaining.size() < key_length) {
                 return errc::network::protocol_error;
             }
-            key = { remaining.begin(), remaining.begin() + static_cast<std::ptrdiff_t>(key_length) };
+            key = { reinterpret_cast<const char*>(remaining.data()), key_length };
             data = gsl::make_span(remaining.data() + key_length, remaining.size() - key_length);
         }
 
@@ -242,19 +267,10 @@ class crud_component_impl
               if (error) {
                   return cb({}, error);
               }
-              bool ids_only;
-              switch (response->extras_.size()) {
-                  case 4:
-                      ids_only = mcbp::big_endian::read_uint32(response->extras_, 0) == 0;
-                      break;
-
-                  case 0:
-                      ids_only = options.ids_only; // support servers before MB-54267. TODO: remove after server GA
-                      break;
-
-                  default:
-                      return cb({}, errc::network::protocol_error);
+              if (response->extras_.size() != 4) {
+                  return cb({}, errc::network::protocol_error);
               }
+              bool ids_only = mcbp::big_endian::read_uint32(response->extras_, 0) == 0;
 
               if (auto ec = parse_range_scan_data(response->value_, std::move(item_cb), ids_only); ec) {
                   return cb({}, ec);
@@ -276,6 +292,19 @@ class crud_component_impl
 
         req->persistent_ = true;
         req->vbucket_ = vbucket_id;
+
+        if (options.timeout != std::chrono::milliseconds::zero()) {
+            auto timer = std::make_shared<asio::steady_timer>(io_);
+            timer->expires_after(options.timeout);
+            timer->async_wait([req](auto error) {
+                if (error == asio::error::operation_aborted) {
+                    return;
+                }
+                req->cancel(couchbase::errc::common::unambiguous_timeout);
+            });
+            req->set_deadline(timer);
+        }
+
         mcbp::buffer_writer buf{ scan_uuid.size() + sizeof(std::uint32_t) * 3 };
         buf.write(scan_uuid);
         buf.write_uint32(options.batch_item_limit);