couchbase 3.0.0.alpha.1-universal-darwin-19 → 3.0.0.beta.1-universal-darwin-19

data/examples/search_with_consistency.rb

@@ -0,0 +1,84 @@
+require 'couchbase'
+
+include Couchbase
+
+options = Cluster::ClusterOptions.new
+options.authenticate("Administrator", "password")
+cluster = Cluster.connect("couchbase://localhost", options)
+
+bucket_name = "default"
+# create index definition, if it does not exist already
+search_index_name = "knob_search"
+begin
+  cluster.search_indexes.get_index(search_index_name)
+rescue Error::IndexNotFound
+  index = Management::SearchIndex.new
+  index.type = "fulltext-index"
+  index.name = search_index_name
+  index.source_type = "couchbase"
+  index.source_name = bucket_name
+  index.params = {
+      mapping: {
+          types: {
+              "knob" => {
+                  properties: {
+                      "name" => {
+                          fields: [
+                              {
+                                  name: "name",
+                                  type: "text",
+                                  include_in_all: true,
+                                  include_term_vectors: true,
+                                  index: true,
+                                  store: true,
+                                  docvalues: true,
+                              }
+                          ]
+                      },
+                  }
+              }
+          }
+      }
+  }
+
+  cluster.search_indexes.upsert_index(index)
+
+  num_indexed = 0
+  loop do
+    sleep(1)
+    num = cluster.search_indexes.get_indexed_documents_count(search_index_name)
+    break if num_indexed == num
+    num_indexed = num
+    puts "indexing #{search_index_name.inspect}: #{num_indexed} documents"
+  end
+end
+
+collection = cluster.bucket(bucket_name).default_collection
+
+# The application need to to know exactly which mutation affect the result,
+# and supply mutation tokens from those operations.
+random_string = ("a".."z").to_a.sample(10).join
+res = collection.upsert("user:#{random_string}", {
+    "name" => "Brass Doorknob",
+    "email" => "brass.doorknob@example.com",
+    "data" => random_string,
+})
+
+state = MutationState.new(res.mutation_token)
+# state.add(*tokens) could be used to add more tokens
+
+query = Cluster::SearchQuery.term("doorknob")
+options = Cluster::SearchOptions.new
+options.timeout = 10_000
+options.consistent_with(state)
+res = cluster.search_query(search_index_name, query, options)
+
+res.rows.each do |row|
+  if row.id == "user:#{random_string}"
+    puts "--- Found our newly created document!"
+  end
+  if ENV['REMOVE_DOOR_KNOBS']
+    puts "Removing #{row.id} (requested via env)"
+    collection.remove(row.id)
+  end
+end

data/examples/subdocument.rb

@@ -7,19 +7,41 @@ cluster = Cluster.connect("couchbase://localhost", options)
 bucket = cluster.bucket("default")
 collection = bucket.default_collection
 
+document = {
+    name: "Douglas Reynholm",
+    email: "douglas@reynholmindustries.com",
+    addresses: {
+        billing: {
+            line1: "123 Any Street",
+            line2: "Anytown",
+            country: "United Kingdom"
+        },
+        delivery: {
+            line1: "123 Any Street",
+            line2: "Anytown",
+            country: "United Kingdom"
+        }
+    },
+    purchases: {
+        complete: [339, 976, 442, 666],
+        abandoned: [157, 42, 999]
+    }
+}
+collection.upsert("customer123", document)
+
 res = collection.mutate_in("customer123", [
-    MutateInSpec.dict_upsert("fax", "311-555-0151")
+    MutateInSpec.upsert("fax", "311-555-0151")
 ])
 puts "The document has been modified successfully: cas=#{res.cas}"
 
 res = collection.mutate_in("customer123", [
-    MutateInSpec.dict_upsert("_framework.model_type", "Customer").xattr,
+    MutateInSpec.upsert("_framework.model_type", "Customer").xattr,
     MutateInSpec.remove("addresses.billing[2]"),
     MutateInSpec.replace("email", "dougr96@hotmail.com"),
 ])
 puts "The document has been modified successfully: cas=#{res.cas}"
 
-res = collection.lookup_in"customer123", [
+res = collection.lookup_in "customer123", [
     LookupInSpec.get("addresses.delivery.country"),
     LookupInSpec.exists("purchases.pending[-1]"),
 ]

data/examples/view.rb

@@ -0,0 +1,50 @@
+# coding: utf-8
+#
+require 'couchbase'
+
+include Couchbase
+
+options = Cluster::ClusterOptions.new
+options.authenticate("Administrator", "password")
+cluster = Cluster.connect("couchbase://localhost", options)
+
+bucket = cluster.bucket("beer-sample")
+collection = bucket.default_collection
+
+options = Bucket::ViewOptions.new
+options.reduce = true
+options.group_level = 1
+res = bucket.view_query("beer", "by_location", options)
+puts "Breweries by country:"
+res.rows.each do |row|
+  puts "#{row.key.first}: #{row.value} breweries"
+end
+
+options = Bucket::ViewOptions.new
+options.limit = 10
+options.order = :descending
+res = bucket.view_query("beer", "brewery_beers", options)
+puts "\nTotal documents in 'beer/brewery_beers' index: #{res.meta_data.total_rows}"
+puts "Last #{options.limit} documents:"
+res.rows.each_with_index do |row|
+  doc = collection.get(row.id)
+  puts "#{row.id} (type: #{doc.content['type']}, key: #{row.key})"
+end
+
+
+random_number = rand(0..1_000_000_000)
+unique_brewery_id = "random_brewery:#{random_number}"
+collection.upsert("random_brewery:#{random_number}", {
+    "name" => "Random brewery: #{random_number}",
+    "type" => "brewery"
+})
+puts "\nRequest with consistency. Generated brewery name: #{unique_brewery_id}"
+options = Bucket::ViewOptions.new
+options.start_key = ["random_brewery:"]
+options.scan_consistency = :request_plus
+res = bucket.view_query("beer", "brewery_beers", options)
+res.rows.each do |row|
+  if row.id == unique_brewery_id
+    puts "Found newly created document: #{collection.get(row.id).content}"
+  end
+end

data/ext/.clang-format

@@ -8,7 +8,7 @@ UseTab: Never
 IndentWidth: 4
 ColumnLimit: 140
 AllowShortFunctionsOnASingleLine: None
-AllowShortBlocksOnASingleLine: false
+AllowShortBlocksOnASingleLine: Empty
 AllowShortCaseLabelsOnASingleLine: false
 BreakBeforeBraces: Linux
 SortIncludes: false

data/ext/.clang-tidy

@@ -17,4 +17,6 @@ Checks: |-
   -cppcoreguidelines-avoid-magic-numbers,
   -cppcoreguidelines-pro-bounds-pointer-arithmetic,
   -modernize-use-trailing-return-type,
+  -llvmlibc-callee-namespace,
+  -abseil-*,
   -fuchsia-*

data/ext/.idea/dictionaries/couchbase_terms.xml

@@ -2,7 +2,9 @@
   <dictionary name="couchbase_terms">
     <words>
       <w>dataverse</w>
+      <w>keyspace</w>
       <w>opcode</w>
+      <w>sasl</w>
       <w>subdoc</w>
       <w>upsert</w>
       <w>vbucket</w>

data/ext/.idea/misc.xml

@@ -1,4 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="CMakeWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
+  <component name="CidrRootsConfiguration">
+    <sourceRoots>
+      <file path="$PROJECT_DIR$/couchbase" />
+    </sourceRoots>
+    <libraryRoots>
+      <file path="$PROJECT_DIR$/third_party" />
+    </libraryRoots>
+    <excludeRoots>
+      <file path="$PROJECT_DIR$/cmake-build-debug" />
+      <file path="$PROJECT_DIR$/cmake-build-debug-system-gcc" />
+    </excludeRoots>
+  </component>
 </project>

data/ext/.idea/vcs.xml

@@ -6,6 +6,7 @@
     <mapping directory="$PROJECT_DIR$/third_party/gsl" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/third_party/http_parser" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/third_party/json" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/third_party/snappy" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/third_party/spdlog" vcs="Git" />
   </component>
 </project>

data/ext/CMakeLists.txt

@@ -11,6 +11,7 @@ endif()
 add_subdirectory(third_party/gsl)
 add_subdirectory(third_party/json)
 add_subdirectory(third_party/spdlog)
+add_subdirectory(third_party/snappy)
 
 include_directories(BEFORE SYSTEM third_party/gsl/include)
 include_directories(BEFORE SYSTEM third_party/asio/asio/include)
@@ -28,7 +29,7 @@ if(MSVC)
     target_compile_options(project_warnings INTERFACE /W4 /WX "/permissive-")
 else()
     option(ONLY_COVERAGE "Build only tests necessary for coverage" FALSE)
-    option(LIBCPP "Build with libc++" FALSE)
+    option(STATIC_STDLIB "Statically link C++ standard library" FALSE)
     option(ENABLE_COVERAGE "Enable coverage reporting for gcc/clang" FALSE)
     option(ENABLE_FUZZERS "Enable fuzz testing tools" FALSE)
 
@@ -44,6 +45,18 @@ else()
         target_link_libraries(project_options INTERFACE -fsanitize=address)
     endif()
 
+    option(ENABLE_TSAN "Enable thread sanitizer" FALSE)
+
+    if(ENABLE_TSAN)
+        target_compile_options(project_options INTERFACE -fsanitize=thread)
+        target_link_libraries(project_options INTERFACE -fsanitize=thread)
+    endif()
+
+    if(STATIC_STDLIB)
+        target_compile_options(project_options INTERFACE -static-libgcc -static-libstdc++)
+        target_link_options(project_options INTERFACE -static-libgcc -static-libstdc++)
+    endif()
+
     target_compile_options(
         project_warnings
         INTERFACE -Wall
@@ -79,6 +92,9 @@ else()
     endif()
 endif()
 
+# Read more at https://wiki.wireshark.org/TLS
+option(TLS_KEY_LOG_FILE "Path to file to write per-session secrets (Useful for Wireshark SSL/TLS dissection)")
+
 include(FindOpenSSL)
 message(STATUS "OPENSSL_VERSION: ${OPENSSL_VERSION}")
 message(STATUS "OPENSSL_INCLUDEDIRS: ${OPENSSL_INCLUDE_DIR}")
@@ -107,9 +123,23 @@ else()
     message(STATUS "RUBY_EXECUTABLE: ${RUBY_EXECUTABLE}")
 endif()
 message(STATUS "RUBY_INCLUDE_DIR: ${RUBY_INCLUDE_DIR}")
+message(STATUS "RUBY_LIBRARY: ${RUBY_LIBRARY}")
 include_directories(BEFORE SYSTEM "${RUBY_INCLUDE_DIR}")
 
+find_program(GIT git)
+if(GIT)
+    execute_process(
+        COMMAND git rev-parse HEAD
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        OUTPUT_VARIABLE BACKEND_GIT_REVISION)
+endif()
+
+string(TIMESTAMP BACKEND_BUILD_TIMESTAMP "%Y-%m-%d %H:%M:%S" UTC)
+configure_file(${PROJECT_SOURCE_DIR}/build_version.hxx.in ${PROJECT_BINARY_DIR}/generated/build_version.hxx @ONLY)
+configure_file(${PROJECT_SOURCE_DIR}/build_config.hxx.in ${PROJECT_BINARY_DIR}/generated/build_config.hxx @ONLY)
 add_library(couchbase SHARED couchbase/couchbase.cxx)
+target_include_directories(couchbase PRIVATE ${PROJECT_BINARY_DIR}/generated)
 target_link_libraries(
     couchbase
     PRIVATE project_options
@@ -120,8 +150,9 @@ target_link_libraries(
             cbcrypto
             cbsasl
             http_parser
+            snappy
             spdlog::spdlog_header_only)
-set_target_properties(cbcrypto cbsasl platform PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+set_target_properties(cbcrypto cbsasl platform snappy PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
 
 if(APPLE)
     target_link_options(couchbase PRIVATE -Wl,-undefined,dynamic_lookup)
@@ -130,16 +161,12 @@ else()
 endif()
 
 if(BUILD_EXAMPLES)
+    file(
+        GENERATE
+        OUTPUT ${PROJECT_BINARY_DIR}/generated/generated_config.hxx
+        CONTENT "#pragma once\n#define LIBCOUCHBASE_EXT_PATH \"$<TARGET_FILE:couchbase>\"")
     add_executable(main test/main.cxx)
-    target_link_libraries(
-        main
-        PRIVATE project_options
-                project_warnings
-                OpenSSL::SSL
-                OpenSSL::Crypto
-                platform
-                cbcrypto
-                cbsasl
-                http_parser
-                spdlog::spdlog_header_only)
+    target_include_directories(main PRIVATE ${PROJECT_BINARY_DIR}/generated)
+    target_link_libraries(main PRIVATE project_options project_warnings ${RUBY_LIBRARY} spdlog::spdlog_header_only)
+    add_dependencies(main couchbase)
 endif()

data/ext/{couchbase/io/binary_message.hxx → build_config.hxx.in}

@@ -17,22 +17,4 @@
 
 #pragma once
 
-namespace couchbase::io
-{
-struct binary_header {
-    uint8_t magic;
-    uint8_t opcode;
-    uint16_t keylen;
-    uint8_t extlen;
-    uint8_t datatype;
-    uint16_t specific;
-    uint32_t bodylen;
-    uint32_t opaque;
-    uint64_t cas;
-};
-
-struct binary_message {
-    binary_header header;
-    std::vector<uint8_t> body;
-};
-} // namespace couchbase::io
+#cmakedefine TLS_KEY_LOG_FILE "@TLS_KEY_LOG_FILE@"

data/ext/build_version.hxx.in

@@ -0,0 +1,26 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#define BACKEND_BUILD_TIMESTAMP "@BACKEND_BUILD_TIMESTAMP@"
+#define BACKEND_CXX_COMPILER "@CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@"
+#define BACKEND_C_COMPILER "@CMAKE_C_COMPILER_ID@ @CMAKE_C_COMPILER_VERSION@"
+#define BACKEND_SYSTEM "@CMAKE_SYSTEM@"
+#define BACKEND_SYSTEM_PROCESSOR "@CMAKE_SYSTEM_PROCESSOR@"
+#define BACKEND_GIT_REVISION "f01d0c237750cc67ee75325ed160bc0da1d2bf7a"
+

data/ext/couchbase/bucket.hxx

@@ -18,58 +18,144 @@
 #pragma once
 
 #include <utility>
+#include <queue>
 
-#include <configuration_monitor.hxx>
 #include <operations.hxx>
+#include <origin.hxx>
 
 namespace couchbase
 {
-
-class bucket
+class bucket : public std::enable_shared_from_this<bucket>
 {
   public:
-    explicit bucket(asio::io_context& ctx, std::string name, configuration config)
-      : ctx_(ctx)
+    explicit bucket(const std::string& client_id,
+                    asio::io_context& ctx,
+                    asio::ssl::context& tls,
+                    std::string name,
+                    couchbase::origin origin,
+                    const std::vector<protocol::hello_feature>& known_features)
+
+      : client_id_(client_id)
+      , ctx_(ctx)
+      , tls_(tls)
       , name_(std::move(name))
-      , config_(std::move(config))
+      , origin_(std::move(origin))
+      , known_features_(known_features)
     {
     }
 
-    [[nodiscard]] const std::string& name() const
+    ~bucket()
     {
-        return name_;
+        close();
     }
 
-    void set_node(size_t index, std::shared_ptr<io::key_value_session> session)
+    [[nodiscard]] const std::string& name() const
     {
-        sessions_.emplace(index, std::move(session));
+        return name_;
     }
 
-    [[nodiscard]] std::shared_ptr<io::key_value_session> get_node(size_t index)
+    template<typename Handler>
+    void bootstrap(Handler&& handler)
     {
-        return sessions_.at(index);
+        std::shared_ptr<io::mcbp_session> new_session;
+        if (origin_.options().enable_tls) {
+            new_session = std::make_shared<io::mcbp_session>(client_id_, ctx_, tls_, origin_, name_, known_features_);
+        } else {
+            new_session = std::make_shared<io::mcbp_session>(client_id_, ctx_, origin_, name_, known_features_);
+        }
+        new_session->bootstrap([self = shared_from_this(), new_session, h = std::forward<Handler>(handler)](
+                                 std::error_code ec, const configuration& cfg) mutable {
+            if (!ec) {
+                self->config_ = cfg;
+                size_t this_index = new_session->index();
+                self->sessions_.emplace(this_index, std::move(new_session));
+                if (cfg.nodes.size() > 1) {
+                    for (const auto& n : cfg.nodes) {
+                        if (n.index != this_index) {
+                            couchbase::origin origin(
+                              self->origin_.get_username(),
+                              self->origin_.get_password(),
+                              n.hostname_for(self->origin_.options().network),
+                              n.port_or(self->origin_.options().network, service_type::kv, self->origin_.options().enable_tls, 0),
+                              self->origin_.options());
+                            std::shared_ptr<io::mcbp_session> s;
+                            if (self->origin_.options().enable_tls) {
+                                s = std::make_shared<io::mcbp_session>(
+                                  self->client_id_, self->ctx_, self->tls_, origin, self->name_, self->known_features_);
+                            } else {
+                                s = std::make_shared<io::mcbp_session>(
+                                  self->client_id_, self->ctx_, origin, self->name_, self->known_features_);
+                            }
+                            s->bootstrap([host = n.hostname, bucket = self->name_](std::error_code err, const configuration& /*config*/) {
+                                // TODO: retry, we know that auth is correct
+                                if (err) {
+                                    spdlog::warn("unable to bootstrap node {} ({}): {}", host, bucket, err.message());
+                                }
+                            });
+                            self->sessions_.emplace(n.index, std::move(s));
+                        }
+                    }
+                }
+                while (!self->deferred_commands_.empty()) {
+                    self->deferred_commands_.front()();
+                    self->deferred_commands_.pop();
+                }
+            }
+            h(ec, cfg);
+        });
     }
 
     template<typename Request, typename Handler>
     void execute(Request request, Handler&& handler)
     {
-        auto cmd = std::make_shared<operations::command<Request>>(ctx_, std::move(request));
-        size_t index = 0;
-        std::tie(cmd->request.partition, index) = config_.map_key(request.id.key);
-        cmd->send_to(sessions_.at(index), std::forward<Handler>(handler));
+        if (closed_) {
+            return;
+        }
+        auto cmd = std::make_shared<operations::mcbp_command<Request>>(ctx_, request);
+        cmd->start([cmd, handler = std::forward<Handler>(handler)](std::error_code ec, std::optional<io::mcbp_message> msg) mutable {
+            using encoded_response_type = typename Request::encoded_response_type;
+            handler(make_response(ec, cmd->request, msg ? encoded_response_type(*msg) : encoded_response_type{}));
+        });
+        if (config_) {
+            map_and_send(cmd);
+        } else {
+            deferred_commands_.emplace([self = shared_from_this(), cmd]() { self->map_and_send(cmd); });
+        }
     }
 
     void close()
     {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
         for (auto& session : sessions_) {
             session.second->stop();
         }
     }
 
+    template<typename Request>
+    void map_and_send(std::shared_ptr<operations::mcbp_command<Request>> cmd)
+    {
+        size_t index = 0;
+        std::tie(cmd->request.partition, index) = config_->map_key(cmd->request.id.key);
+        auto session = sessions_.at(index);
+        cmd->send_to(session);
+    }
+
   private:
+    std::string client_id_;
     asio::io_context& ctx_;
+    asio::ssl::context& tls_;
     std::string name_;
-    configuration config_;
-    std::map<size_t, std::shared_ptr<io::key_value_session>> sessions_;
+    origin origin_;
+
+    std::optional<configuration> config_{};
+    std::vector<protocol::hello_feature> known_features_;
+
+    std::queue<std::function<void()>> deferred_commands_{};
+
+    bool closed_{ false };
+    std::map<size_t, std::shared_ptr<io::mcbp_session>> sessions_{};
 };
-} // namespace couchbase
+} // namespace couchbase