couchbase 4.2.5 → 4.2.6-dev.1

package/deps/couchbase-cxx-client/CMakeLists.txt

@@ -100,6 +100,7 @@ set(couchbase_cxx_client_FILES
         core/free_form_http_request.cxx
         core/key_value_config.cxx
         core/n1ql_query_options.cxx
+        core/origin.cxx
         core/range_scan_options.cxx
         core/range_scan_orchestrator.cxx
         core/retry_orchestrator.cxx
@@ -265,10 +266,10 @@ set(couchbase_cxx_client_FILES
         core/impl/boolean_query.cxx
         core/impl/build_deferred_query_indexes.cxx
         core/impl/cluster.cxx
-        core/impl/collection_query_index_manager.cxx
         core/impl/common_error_category.cxx
         core/impl/configuration_profiles_registry.cxx
         core/impl/conjunction_query.cxx
+        core/impl/create_bucket.cxx
         core/impl/create_query_index.cxx
         core/impl/date_range.cxx
         core/impl/date_range_facet.cxx
@@ -277,24 +278,29 @@ set(couchbase_cxx_client_FILES
         core/impl/decrement.cxx
         core/impl/disjunction_query.cxx
         core/impl/dns_srv_tracker.cxx
+        core/impl/drop_bucket.cxx
         core/impl/drop_query_index.cxx
         core/impl/exists.cxx
         core/impl/expiry.cxx
         core/impl/fail_fast_retry_strategy.cxx
         core/impl/field_level_encryption_error_category.cxx
+        core/impl/flush_bucket.cxx
         core/impl/geo_bounding_box_query.cxx
         core/impl/geo_distance_query.cxx
         core/impl/geo_polygon_query.cxx
         core/impl/get.cxx
+        core/impl/get_all_buckets.cxx
         core/impl/get_all_query_indexes.cxx
         core/impl/get_all_replicas.cxx
         core/impl/get_and_lock.cxx
         core/impl/get_and_touch.cxx
         core/impl/get_any_replica.cxx
+        core/impl/get_bucket.cxx
         core/impl/get_replica.cxx
         core/impl/increment.cxx
         core/impl/insert.cxx
         core/impl/internal_date_range_facet_result.cxx
+        core/impl/internal_manager_error_context.cxx
         core/impl/internal_numeric_range_facet_result.cxx
         core/impl/internal_search_error_context.cxx
         core/impl/internal_search_meta_data.cxx
@@ -306,6 +312,7 @@ set(couchbase_cxx_client_FILES
         core/impl/key_value_error_context.cxx
         core/impl/lookup_in.cxx
         core/impl/management_error_category.cxx
+        core/impl/manager_error_context.cxx
         core/impl/match_all_query.cxx
         core/impl/match_none_query.cxx
         core/impl/match_phrase_query.cxx
@@ -368,6 +375,7 @@ set(couchbase_cxx_client_FILES
         core/impl/transaction_get_result.cxx
         core/impl/transaction_op_error_category.cxx
         core/impl/unlock.cxx
+        core/impl/update_bucket.cxx
         core/impl/upsert.cxx
         core/impl/view_error_category.cxx
         core/impl/watch_query_indexes.cxx

package/deps/couchbase-cxx-client/bin/api.rb

@@ -0,0 +1,234 @@
+#  Copyright 2020-2021 Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require "openssl"
+require "net/http"
+require "json"
+
+class Object
+  def to_b
+    ![nil, false, 0, "", "0", "f", "F", "false", "FALSE", "off", "OFF", "no", "NO"].include?(self)
+  end
+end
+
+
+class API
+  def initialize(options)
+    @options = options
+    connect
+  end
+
+  def connect
+    puts "# CONNECT #{@options}"
+    @client = Net::HTTP.start(@options[:host], @options[:port],
+                              use_ssl: @options[:strict_encryption],
+                              verify_mode: OpenSSL::SSL::VERIFY_NONE)
+  end
+
+  def url(path)
+    "#{@client.use_ssl? ? 'https' : 'http'}://#{@options[:username]}@#{@options[:host]}:#{@options[:port]}#{path}"
+  end
+
+  def decode_response(response)
+    payload =
+      if response['content-type'] =~ /application\/json/
+        JSON.parse(response.body)
+      else
+        response.body
+      end
+    if @options[:verbose]
+      p status: response.code, payload: payload
+    end
+    payload
+  end
+
+  def setup_request(request)
+    request.basic_auth(@options[:username], @options[:password])
+    request['accept'] = "application/json"
+  end
+
+  def get(path)
+    puts "# GET #{url(path)}"
+    req = Net::HTTP::Get.new(path)
+    setup_request(req)
+    res = @client.request(req)
+    decode_response(res)
+  rescue EOFError
+    connect
+    retry
+  rescue => ex
+    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+
+  def post_form(path, fields = {})
+    puts "# POST #{url(path)} #{fields}"
+    req = Net::HTTP::Post.new(path)
+    setup_request(req)
+    req.form_data = fields
+    res = @client.request(req)
+    decode_response(res)
+  rescue EOFError
+    connect
+    retry
+  rescue => ex
+    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+
+  def post_json(path, object)
+    data = JSON.generate(object)
+    puts "# POST #{url(path)} #{data}"
+    req = Net::HTTP::Post.new(path)
+    req['content-type'] = "application/json"
+    setup_request(req)
+    res = @client.request(req, data)
+    decode_response(res)
+  rescue EOFError
+    connect
+    retry
+  rescue => ex
+    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+
+  def put_json(path, object)
+    data = JSON.generate(object)
+    puts "# PUT #{url(path)} #{data}"
+    req = Net::HTTP::Put.new(path)
+    req['content-type'] = "application/json"
+    setup_request(req)
+    res = @client.request(req, data)
+    decode_response(res)
+  rescue EOFError
+    connect
+    retry
+  rescue => ex
+    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+
+  def delete(path)
+    puts "# DELETE #{url(path)}"
+    req = Net::HTTP::Delete.new(path)
+    setup_request(req)
+    res = @client.request(req)
+    decode_response(res)
+  rescue EOFError
+    connect
+    retry
+  rescue => ex
+    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+end
+
+def service_address_for_bucket(api, service, bucket, options)
+  port_key = options[:strict_encryption] ? "#{service}SSL" : service
+  host = nil
+  port = 0
+  while port.zero?
+    config = api.get("/pools/default/b/#{bucket}")
+    node_with_service = config["nodesExt"].find{|n| n["services"].key?(port_key)} rescue nil
+    if node_with_service
+      host = node_with_service["hostname"]
+      port = node_with_service["services"][port_key].to_i rescue 0
+    end
+    sleep 1
+  end
+ {host: host || options[:host], port: port}
+end
+
+def ensure_search_index_created(index_definition_path, options)
+  loop do
+    catch :retry do
+      search_api = API.new(options)
+
+      puts "using index definition: #{File.basename(index_definition_path)}"
+      index_definition = JSON.load(File.read(index_definition_path))
+      search_api.put_json("/api/index/travel-sample-index", index_definition)
+
+      expected_number_of_the_documents = 1000
+      indexed_documents = 0
+      times_zero_documents_has_been_seen = 0
+      Timeout.timeout(10 * 60) do # give it 10 minutes to index
+        while indexed_documents < expected_number_of_the_documents
+          resp = search_api.get("/api/index/travel-sample-index/count")
+          indexed_documents = resp['count'].to_i
+          times_zero_documents_has_been_seen += 1 if indexed_documents.zero?
+          if times_zero_documents_has_been_seen >= 60
+            # lets not tolerate 60 seconds of not getting data indexed
+            search_api.delete("/api/index/travel-sample-index")
+            throw :retry
+          end
+          sleep 1
+        end
+      end
+      return
+    end
+  end
+end
+
+def ensure_sample_bucket_loaded(management_api, expected_number_of_the_documents, options)
+  loop do
+    catch :retry do
+      management_api.post_json("/sampleBuckets/install", [options[:bucket]])
+
+      service_address = nil
+      begin
+        Timeout.timeout(60) do # a minute for bucket creation
+          service_address = service_address_for_bucket(management_api, "n1ql", options[:bucket], options)
+        end
+      rescue Timeout::Error
+        management_api.delete("/pools/default/buckets/#{options[:bucket]}")
+        sleep 1
+        throw :retry
+      end
+      puts query_service: service_address
+
+      query_api = API.new(options.merge(service_address))
+      puts "create index for bucket \"#{options[:bucket]}\""
+
+      query_api.post_form("/query/service", statement: "CREATE PRIMARY INDEX ON `#{options[:bucket]}` USING GSI", timeout: "300s")
+
+      indexed_documents = 0
+      times_zero_documents_has_been_seen = 0
+      Timeout.timeout(10 * 60) do # give it 10 minutes to index
+        while indexed_documents < expected_number_of_the_documents
+          resp = query_api.post_form("/query/service", statement: "SELECT RAW COUNT(*) FROM `#{options[:bucket]}`")
+          indexed_documents = resp["results"][0].to_i rescue 0
+
+          if indexed_documents.zero?
+            times_zero_documents_has_been_seen += 1
+            puts times_zero_documents_has_been_seen: times_zero_documents_has_been_seen
+          end
+          if times_zero_documents_has_been_seen >= 60
+            # lets not tolerate 60 seconds of not getting data indexed
+            query_api.post_form("/query/service", statement: "DROP PRIMARY INDEX ON `#{options[:bucket]}`")
+            management_api.delete("/pools/default/buckets/#{options[:bucket]}")
+            sleep 1
+            throw :retry
+          end
+          sleep 1
+        end
+      end
+      return
+    end
+  end
+end

package/deps/couchbase-cxx-client/bin/create-search-index

@@ -15,150 +15,33 @@
 #  limitations under the License.
 
 require "timeout"
-require "net/http"
-require "json"
-require "openssl"
 
-class API
-  def initialize(options)
-    @options = options
-    connect
-  end
-
-  def connect
-    puts "# CONNECT #{@options}"
-    @client = Net::HTTP.start(@options[:host], @options[:port],
-                              use_ssl: @options[:strict_encryption],
-                              verify_mode: OpenSSL::SSL::VERIFY_NONE)
-  end
-
-  def url(path)
-    "#{@client.use_ssl? ? 'https' : 'http'}://#{@options[:username]}@#{@options[:host]}:#{@options[:port]}#{path}"
-  end
-
-  def decode_response(response)
-    payload =
-      if response['content-type'] =~ /application\/json/
-        JSON.parse(response.body)
-      else
-        response.body
-      end
-    p payload if @options[:verbose]
-    payload
-  end
-
-  def setup_request(request)
-    request.basic_auth(@options[:username], @options[:password])
-    request['accept'] = "application/json"
-  end
-
-  def get(path)
-    puts "# GET #{url(path)}"
-    req = Net::HTTP::Get.new(path)
-    setup_request(req)
-    res = @client.request(req)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def post_form(path, fields = {})
-    puts "# POST #{url(path)} #{fields}"
-    req = Net::HTTP::Post.new(path)
-    setup_request(req)
-    req.form_data = fields
-    res = @client.request(req)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def post_json(path, object)
-    data = JSON.generate(object)
-    puts "# POST #{url(path)} #{data}"
-    req = Net::HTTP::Post.new(path)
-    req['content-type'] = "application/json"
-    setup_request(req)
-    res = @client.request(req, data)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def put_json(path, object)
-    data = JSON.generate(object)
-    puts "# PUT #{url(path)} #{data}"
-    req = Net::HTTP::Put.new(path)
-    req['content-type'] = "application/json"
-    setup_request(req)
-    res = @client.request(req, data)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-end
-
-def service_port_for_bucket(api, service, bucket, options)
-  port_key = options[:strict_encryption] ? "#{service}SSL" : service
-  port = 0
-  while port.zero?
-    config = api.get("/pools/default/b/#{bucket}")
-    port = config["nodesExt"].find{|n| n["services"].key?(port_key)}["services"][port_key].to_i rescue 0
-    sleep 1
-  end
-  port
-end
+require_relative "api"
 
 options = {
-  host: ARGV[0] || "127.0.0.1",
-  strict_encryption: ARGV[1] == "true" || ARGV[1] == "1",
-  username: ARGV[2] || "Administrator",
-  password: ARGV[3] || "password",
+  host: ENV.fetch("CB_HOST", ARGV[0] || "127.0.0.1"),
+  strict_encryption: ENV.fetch("CB_STRICT_ENCRYPTION", ARGV[1]).to_b,
+  username: ENV.fetch("CB_USERNAME", ARGV[2] || "Administrator"),
+  password: ENV.fetch("CB_PASSWORD", ARGV[3] || "password"),
+  verbose: ENV.fetch("CB_VERBOSE", true).to_b,
   bucket: "travel-sample",
-  verbose: true,
 }
 p options: options
 
-management_api = API.new(options.merge(port: options[:strict_encryption] ? 18091 : 8091))
-search_service_port = service_port_for_bucket(management_api, "fts", options[:bucket], options)
+management_api =
+  begin
+    API.new(options.merge(port: options[:strict_encryption] ? 18091 : 8091))
+  rescue => ex
+    puts "#{ex}, sleep for 1 second and retry"
+    sleep(1)
+    retry
+  end
+service_address = service_address_for_bucket(management_api, "fts", options[:bucket], options)
+p search_service: service_address
 
-has_v6_nodes = 
+has_v6_nodes =
   management_api.get("/pools/default")["nodes"]
   .any? { |node| node["version"] =~ /^6\./ && node["services"].include?("fts") }
 
-search_api = API.new(options.merge(port: search_service_port))
 index_definition_path = File.join(__dir__, "travel-sample-index#{"-v6" if has_v6_nodes}.json")
-puts "using index definition: #{File.basename(index_definition_path)}"
-index_definition = JSON.load(File.read(index_definition_path))
-search_api.put_json("/api/index/travel-sample-index", index_definition)
-
-expected_number_of_the_documents = 1000
-indexed_documents = 0
-
-Timeout.timeout(10 * 60) do # give it 10 minutes to index
-  while indexed_documents < expected_number_of_the_documents
-    resp = search_api.get("/api/index/travel-sample-index/count")
-    indexed_documents = resp['count'].to_i
-    sleep 1
-  end
-end
+ensure_search_index_created(index_definition_path, options.merge(service_address))

package/deps/couchbase-cxx-client/bin/init-cluster

@@ -17,13 +17,8 @@
 require "set"
 require "timeout"
 require "optparse"
-require "openssl"
 
-class Object
-  def to_b
-    ![nil, false, 0, "", "0", "f", "F", "false", "FALSE", "off", "OFF", "no", "NO"].include?(self)
-  end
-end
+require_relative "api"
 
 options = {
   verbose: ENV.fetch("CB_VERBOSE", true).to_b,
@@ -49,106 +44,6 @@ options[:port] = ENV.fetch("CB_PORT", default_port).to_i
 options[:sample_buckets] << "beer-sample" if ENV.fetch("CB_BEER_SAMPLE", false).to_b
 options[:sample_buckets] << "travel-sample" if ENV.fetch("CB_TRAVEL_SAMPLE", false).to_b
 
-require "net/http"
-require "json"
-
-class API
-  def initialize(options)
-    @options = options
-    connect
-  end
-
-  def connect
-    @client = Net::HTTP.start(@options[:host], @options[:port],
-                              use_ssl: @options[:strict_encryption],
-                              verify_mode: OpenSSL::SSL::VERIFY_NONE)
-  end
-
-  def url(path)
-    "#{@client.use_ssl? ? 'https' : 'http'}://#{@options[:username]}@#{@options[:host]}:#{@options[:port]}#{path}"
-  end
-
-  def decode_response(response)
-    payload =
-      if /application\/json/.match?(response['content-type'])
-        JSON.parse(response.body)
-      else
-        response.body
-      end
-    p payload if @options[:verbose]
-    payload
-  end
-
-  def setup_request(request)
-    request.basic_auth(@options[:username], @options[:password])
-    request['accept'] = "application/json"
-  end
-
-  def get(path)
-    puts "# GET #{url(path)}"
-    req = Net::HTTP::Get.new(path)
-    setup_request(req)
-    res = @client.request(req)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def post_form(path, fields = {})
-    puts "# POST #{url(path)} #{fields}"
-    req = Net::HTTP::Post.new(path)
-    setup_request(req)
-    req.form_data = fields
-    res = @client.request(req)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def post_json(path, object)
-    data = JSON.generate(object)
-    puts "# POST #{url(path)} #{data}"
-    req = Net::HTTP::Post.new(path)
-    req['content-type'] = "application/json"
-    setup_request(req)
-    res = @client.request(req, data)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-
-  def put_json(path, object)
-    data = JSON.generate(object)
-    puts "# PUT #{url(path)} #{data}"
-    req = Net::HTTP::Put.new(path)
-    req['content-type'] = "application/json"
-    setup_request(req)
-    res = @client.request(req, data)
-    decode_response(res)
-  rescue EOFError
-    connect
-    retry
-  rescue => ex
-    puts "#{__method__}: #{ex}, sleep for 1 second and retry"
-    sleep(1)
-    retry
-  end
-end
 
 api = 
   begin
@@ -180,7 +75,10 @@ api.post_form("/settings/web",
               password: options[:password],
               username: options[:username],
               port: "SAME")
-api.post_form("/settings/indexes", storageMode: "plasma")
+res = api.post_form("/settings/indexes", storageMode: "plasma")
+if res["errors"]
+  res = api.post_form("/settings/indexes", storageMode: "forestdb")
+end
 
 if options[:cluster_run_nodes] > 1
   known_nodes = []
@@ -235,50 +133,30 @@ if options[:sec_bucket].to_b
               name: options[:sec_bucket])
 end
 
-api.post_json("/sampleBuckets/install", options[:sample_buckets].to_a) if options[:sample_buckets].any?
-
 api.post_form("/settings/developerPreview", enabled: true) if options[:enable_developer_preview]
 
-def service_port_for_bucket(api, service, bucket, options)
-  port_key = options[:strict_encryption] ? "#{service}SSL" : service
-  port = 0
-  while port.zero?
-    config = api.get("/pools/default/b/#{bucket}")
-    port = config["nodesExt"].find{|n| n["services"].key?(port_key)}["services"][port_key].to_i rescue 0
-    sleep 1
-  end
-  port
-end
+service_address = service_address_for_bucket(api, "n1ql", options[:bucket], options)
+puts query_service: service_address
 
-query_api = API.new(options.merge(port: service_port_for_bucket(api, "n1ql", options[:bucket], options)))
+query_api = API.new(options.merge(service_address))
 query_api.post_form("/query/service", statement: "CREATE PRIMARY INDEX ON `#{options[:bucket]}` USING GSI")
+
+expected_counts = {
+  "travel-sample" => 30_000,
+  "beer-sample" => 7_000,
+}
 options[:sample_buckets].each do |bucket|
-  service_port_for_bucket(api, "n1ql", bucket, options)
-  query_api.post_form("/query/service", statement: "CREATE PRIMARY INDEX ON `#{bucket}` USING GSI")
+  ensure_sample_bucket_loaded(api, expected_counts[bucket], options.merge(bucket: bucket))
 end
 
-puts service_port_for_bucket(api, "fts", options[:bucket], options)
-
 if options[:sample_buckets].include?("travel-sample") && services.include?("fts")
-  search_api = API.new(options.merge(port: service_port_for_bucket(api, "fts", options[:bucket], options)))
+  service_address = service_address_for_bucket(api, "fts", options[:bucket], options)
+  puts search_service: service_address
 
   has_v6_nodes = 
     api.get("/pools/default")["nodes"]
     .any? { |node| node["version"] =~ /^6\./ && node["services"].include?("fts") }
-
   index_definition_path = File.join(__dir__, "travel-sample-index#{"-v6" if has_v6_nodes}.json")
-  puts "using index definition: #{File.basename(index_definition_path)}"
-  index_definition = JSON.load(File.read(index_definition_path))
-  search_api.put_json("/api/index/travel-sample-index", index_definition)
-
-  expected_number_of_the_documents = 1000
-  indexed_documents = 0
-  Timeout.timeout(10 * 60) do # give it 10 minutes to index
-    while indexed_documents < expected_number_of_the_documents
-      resp = search_api.get("/api/index/travel-sample-index/count")
-      indexed_documents = resp['count'].to_i
-      sleep 1
-    end
-  end
+  ensure_search_index_created(index_definition_path, options.merge(service_address))
 end