couchbase 3.0.0.alpha.1-universal-darwin-19 → 3.0.0.beta.1-universal-darwin-19

data/ext/couchbase/protocol/client_opcode.hxx

@@ -17,23 +17,30 @@
 
 #pragma once
 
+#include <io/mcbp_message.hxx>
+
 namespace couchbase::protocol
 {
-static const size_t header_size = 24;
-using header_buffer = std::array<std::uint8_t, header_size>;
-
 enum class client_opcode : uint8_t {
     get = 0x00,
     upsert = 0x01,
     insert = 0x02,
     replace = 0x03,
     remove = 0x04,
+    increment = 0x05,
+    decrement = 0x06,
+    touch = 0x1c,
+    get_and_touch = 0x1d,
     hello = 0x1f,
     sasl_list_mechs = 0x20,
     sasl_auth = 0x21,
     sasl_step = 0x22,
     select_bucket = 0x89,
+    observe = 0x92,
+    get_and_lock = 0x94,
+    unlock = 0x95,
     get_collections_manifest = 0xba,
+    get_collection_id = 0xbb,
     subdoc_multi_lookup = 0xd0,
     subdoc_multi_mutation = 0xd1,
     get_cluster_config = 0xb5,
@@ -45,6 +52,8 @@ enum class client_opcode : uint8_t {
  * subdocument opcodes are listed separately, because we are not going to implement/support single-op messages
  */
 enum class subdoc_opcode : uint8_t {
+    get_doc = 0x00,
+    set_doc = 0x01,
     get = 0xc5,
     exists = 0xc6,
     dict_add = 0xc7,
@@ -79,6 +88,14 @@ is_valid_client_opcode(uint8_t code)
         case client_opcode::get_error_map:
         case client_opcode::invalid:
         case client_opcode::get_collections_manifest:
+        case client_opcode::touch:
+        case client_opcode::observe:
+        case client_opcode::get_and_lock:
+        case client_opcode::unlock:
+        case client_opcode::get_and_touch:
+        case client_opcode::increment:
+        case client_opcode::decrement:
+        case client_opcode::get_collection_id:
             return true;
     }
     return false;
@@ -100,6 +117,8 @@ is_valid_subdoc_opcode(uint8_t code)
         case subdoc_opcode::array_add_unique:
         case subdoc_opcode::counter:
         case subdoc_opcode::get_count:
+        case subdoc_opcode::get_doc:
+        case subdoc_opcode::set_doc:
             return true;
     }
     return false;
@@ -161,6 +180,30 @@ struct fmt::formatter<couchbase::protocol::client_opcode> : formatter<string_vie
             case couchbase::protocol::client_opcode::get_collections_manifest:
                 name = "get_collections_manifest";
                 break;
+            case couchbase::protocol::client_opcode::touch:
+                name = "touch";
+                break;
+            case couchbase::protocol::client_opcode::observe:
+                name = "observe";
+                break;
+            case couchbase::protocol::client_opcode::get_and_lock:
+                name = "get_and_lock";
+                break;
+            case couchbase::protocol::client_opcode::unlock:
+                name = "unlock";
+                break;
+            case couchbase::protocol::client_opcode::get_and_touch:
+                name = "get_and_touch";
+                break;
+            case couchbase::protocol::client_opcode::increment:
+                name = "increment";
+                break;
+            case couchbase::protocol::client_opcode::decrement:
+                name = "decrement";
+                break;
+            case couchbase::protocol::client_opcode::get_collection_id:
+                name = "get_collection_uid";
+                break;
         }
         return formatter<string_view>::format(name, ctx);
     }
@@ -209,6 +252,12 @@ struct fmt::formatter<couchbase::protocol::subdoc_opcode> : formatter<string_vie
             case couchbase::protocol::subdoc_opcode::get_count:
                 name = "get_count";
                 break;
+            case couchbase::protocol::subdoc_opcode::get_doc:
+                name = "get_doc";
+                break;
+            case couchbase::protocol::subdoc_opcode::set_doc:
+                name = "set_doc";
+                break;
         }
         return formatter<string_view>::format(name, ctx);
     }

data/ext/couchbase/protocol/client_request.hxx

@@ -21,6 +21,8 @@
 #include <arpa/inet.h>
 #endif
 
+#include <snappy.h>
+
 #include <gsl/gsl_util>
 #include <protocol/client_opcode.hxx>
 #include <protocol/magic.hxx>
@@ -37,12 +39,12 @@ class client_request
     using response_type = client_response<response_body_type>;
 
   private:
-    static const size_t header_size = 24;
-    static const magic magic_ = magic::client_request;
+    magic magic_{ magic::client_request };
 
     client_opcode opcode_{ Body::opcode };
     std::uint16_t partition_{ 0 };
     std::uint32_t opaque_{ 0 };
+    std::uint64_t cas_{ 0 };
     Body body_;
     std::vector<std::uint8_t> payload_;
 
@@ -57,6 +59,11 @@ class client_request
         opaque_ = val;
     }
 
+    void cas(std::uint64_t val)
+    {
+        cas_ = val;
+    }
+
     std::uint32_t opaque()
     {
         return opaque_;
@@ -77,23 +84,42 @@ class client_request
         return body_;
     }
 
-    std::vector<std::uint8_t>& data()
+    std::vector<std::uint8_t>& data(bool try_to_compress = false)
     {
-        write_payload();
+        switch (opcode_) {
+            case protocol::client_opcode::insert:
+            case protocol::client_opcode::upsert:
+            case protocol::client_opcode::replace:
+                write_payload(try_to_compress);
+                break;
+            default:
+                write_payload(false);
+                break;
+        }
         return payload_;
     }
 
   private:
-    void write_payload()
+    void write_payload(bool try_to_compress)
     {
         payload_.resize(header_size + body_.size(), 0);
         payload_[0] = static_cast<uint8_t>(magic_);
         payload_[1] = static_cast<uint8_t>(opcode_);
 
-        uint16_t key_size = htons(gsl::narrow_cast<uint16_t>(body_.key().size()));
-        memcpy(payload_.data() + 2, &key_size, sizeof(key_size));
+        auto framing_extras = body_.framing_extras();
+
+        uint16_t key_size = gsl::narrow_cast<uint16_t>(body_.key().size());
+        if (framing_extras.size() == 0) {
+            key_size = htons(key_size);
+            memcpy(payload_.data() + 2, &key_size, sizeof(key_size));
+        } else {
+            magic_ = protocol::magic::alt_client_request;
+            payload_[0] = static_cast<uint8_t>(magic_);
+            payload_[2] = gsl::narrow_cast<std::uint8_t>(framing_extras.size());
+            payload_[3] = gsl::narrow_cast<std::uint8_t>(key_size);
+        }
 
-        uint8_t ext_size = gsl::narrow_cast<uint8_t>(body_.extension().size());
+        uint8_t ext_size = gsl::narrow_cast<uint8_t>(body_.extras().size());
         memcpy(payload_.data() + 4, &ext_size, sizeof(ext_size));
 
         uint16_t vbucket = ntohs(gsl::narrow_cast<uint16_t>(partition_));
@@ -103,10 +129,31 @@ class client_request
         memcpy(payload_.data() + 8, &body_size, sizeof(body_size));
 
         memcpy(payload_.data() + 12, &opaque_, sizeof(opaque_));
+        memcpy(payload_.data() + 16, &cas_, sizeof(cas_));
 
         auto body_itr = payload_.begin() + header_size;
-        body_itr = std::copy(body_.extension().begin(), body_.extension().end(), body_itr);
+        if (framing_extras.size() > 0) {
+            body_itr = std::copy(framing_extras.begin(), framing_extras.end(), body_itr);
+        }
+        body_itr = std::copy(body_.extras().begin(), body_.extras().end(), body_itr);
         body_itr = std::copy(body_.key().begin(), body_.key().end(), body_itr);
+
+        static const std::size_t min_size_to_compress = 32;
+        static const double min_ratio = 0.83;
+        if (try_to_compress && body_.value().size() > min_size_to_compress) {
+            std::string compressed;
+            std::size_t compressed_size =
+              snappy::Compress(reinterpret_cast<const char*>(body_.value().data()), body_.value().size(), &compressed);
+            if (gsl::narrow_cast<double>(compressed_size) / gsl::narrow_cast<double>(body().value().size()) < min_ratio) {
+                std::copy(compressed.begin(), compressed.end(), body_itr);
+                payload_[5] |= static_cast<uint8_t>(protocol::datatype::snappy);
+                size_t new_body_size = body_.size() - (body_.value().size() - compressed_size);
+                body_size = htonl(gsl::narrow_cast<uint32_t>(new_body_size));
+                memcpy(payload_.data() + 8, &body_size, sizeof(body_size));
+                payload_.resize(header_size + new_body_size);
+                return;
+            }
+        }
         std::copy(body_.value().begin(), body_.value().end(), body_itr);
     }
 };

data/ext/couchbase/protocol/client_response.hxx

@@ -26,6 +26,7 @@
 #include <protocol/status.hxx>
 #include <protocol/datatype.hxx>
 #include <protocol/cmd_info.hxx>
+#include <protocol/frame_info_id.hxx>
 
 namespace couchbase::protocol
 {
@@ -39,27 +40,29 @@ template<typename Body>
 class client_response
 {
   private:
-    static const inline magic magic_ = magic::client_response;
-
     Body body_;
+    magic magic_{ magic::client_response };
     client_opcode opcode_{ client_opcode::invalid };
-    header_buffer header_;
-    uint8_t data_type_;
-    std::vector<std::uint8_t> data_;
-    std::size_t body_size_;
-    protocol::status status_;
+    header_buffer header_{};
+    uint8_t data_type_{ 0 };
+    std::vector<std::uint8_t> data_{};
+    std::uint16_t key_size_{ 0 };
+    std::uint8_t framing_extras_size_{ 0 };
+    std::uint8_t extras_size_{ 0 };
+    std::size_t body_size_{ 0 };
+    protocol::status status_{};
     std::optional<enhanced_error> error_;
-    std::uint32_t opaque_;
-    std::uint64_t cas_;
-    cmd_info info_;
+    std::uint32_t opaque_{};
+    std::uint64_t cas_{};
+    cmd_info info_{};
 
   public:
     client_response() = default;
-    explicit client_response(io::binary_message& msg)
+    explicit client_response(io::mcbp_message& msg)
     {
-        std::memcpy(header_.data(), &msg.header, sizeof(msg.header));
-        verify_header();
+        header_ = msg.header_data();
         data_ = std::move(msg.body);
+        verify_header();
         parse_body();
     }
 
@@ -105,8 +108,10 @@ class client_response
 
     void verify_header()
     {
-        Expects(header_[0] == static_cast<std::uint8_t>(magic_));
+        Expects(header_[0] == static_cast<std::uint8_t>(magic::alt_client_response) ||
+                header_[0] == static_cast<std::uint8_t>(magic::client_response));
         Expects(header_[1] == static_cast<std::uint8_t>(Body::opcode));
+        magic_ = static_cast<magic>(header_[0]);
         opcode_ = static_cast<client_opcode>(header_[1]);
         data_type_ = header_[5];
 
@@ -116,6 +121,15 @@ class client_response
         Expects(protocol::is_valid_status(status));
         status_ = static_cast<protocol::status>(status);
 
+        extras_size_ = header_[4];
+        if (magic_ == magic::alt_client_response) {
+            framing_extras_size_ = header_[2];
+            key_size_ = header_[3];
+        } else {
+            memcpy(&key_size_, header_.data() + 2, sizeof(key_size_));
+            key_size_ = ntohs(key_size_);
+        }
+
         uint32_t field = 0;
         memcpy(&field, header_.data() + 8, sizeof(field));
         body_size_ = ntohl(field);
@@ -136,9 +150,10 @@ class client_response
 
     void parse_body()
     {
-        bool parsed = body_.parse(status_, header_, data_, info_);
+        parse_framing_extras();
+        bool parsed = body_.parse(status_, header_, framing_extras_size_, key_size_, extras_size_, data_, info_);
         if (status_ != protocol::status::success && !parsed && has_json_datatype(data_type_)) {
-            auto error = tao::json::from_string(std::string(data_.begin(), data_.end()));
+            auto error = tao::json::from_string(std::string(data_.begin() + framing_extras_size_ + extras_size_ + key_size_, data_.end()));
             if (error.is_object()) {
                 auto& err_obj = error["error"];
                 if (err_obj.is_object()) {
@@ -157,6 +172,28 @@ class client_response
         }
     }
 
+    void parse_framing_extras()
+    {
+        if (framing_extras_size_ == 0) {
+            return;
+        }
+        size_t offset = 0;
+        while (offset < framing_extras_size_) {
+            std::uint8_t frame_size = data_[offset] & 0xfU;
+            std::uint8_t frame_id = (static_cast<std::uint32_t>(data_[offset]) >> 4U) & 0xfU;
+            offset++;
+            if (frame_id == static_cast<std::uint8_t>(response_frame_info_id::server_duration)) {
+                if (frame_size == 2 && framing_extras_size_ - offset >= frame_size) {
+                    std::uint16_t encoded_duration{};
+                    std::memcpy(&encoded_duration, data_.data() + offset, sizeof(encoded_duration));
+                    encoded_duration = ntohs(encoded_duration);
+                    info_.server_duration_us = std::pow(encoded_duration, 1.74) / 2;
+                }
+            }
+            offset += frame_size;
+        }
+    }
+
     [[nodiscard]] std::vector<std::uint8_t>& data()
     {
         return data_;

data/ext/couchbase/protocol/cmd_cluster_map_change_notification.hxx

@@ -0,0 +1,76 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <gsl/gsl_assert>
+
+#include <protocol/server_opcode.hxx>
+#include <protocol/cmd_info.hxx>
+
+#include <configuration.hxx>
+
+namespace couchbase::protocol
+{
+
+class cluster_map_change_notification_request_body
+{
+  public:
+    static const inline server_opcode opcode = server_opcode::cluster_map_change_notification;
+
+  private:
+    uint32_t protocol_revision_;
+    std::string bucket_;
+    configuration config_;
+
+  public:
+    [[nodiscard]] uint32_t protocol_revision()
+    {
+        return protocol_revision_;
+    }
+
+    [[nodiscard]] const std::string& bucket()
+    {
+        return bucket_;
+    }
+
+    [[nodiscard]] configuration config()
+    {
+        return config_;
+    }
+
+    bool parse(const header_buffer& header, const std::vector<uint8_t>& body, const cmd_info&)
+    {
+        Expects(header[1] == static_cast<uint8_t>(opcode));
+        using offset_type = std::vector<uint8_t>::difference_type;
+        uint8_t ext_size = header[4];
+        offset_type offset = ext_size;
+        if (ext_size == 4) {
+            memcpy(&protocol_revision_, body.data(), sizeof(protocol_revision_));
+            protocol_revision_ = ntohl(protocol_revision_);
+        }
+        uint16_t key_size = 0;
+        memcpy(&key_size, header.data() + 2, sizeof(key_size));
+        key_size = ntohs(key_size);
+        bucket_.assign(body.begin() + offset, body.begin() + offset + key_size);
+        offset += key_size;
+        config_ = tao::json::from_string<deduplicate_keys>(std::string(body.begin() + offset, body.end())).as<configuration>();
+        return true;
+    }
+};
+
+} // namespace couchbase::protocol

data/ext/couchbase/protocol/cmd_decrement.hxx

@@ -0,0 +1,187 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <protocol/unsigned_leb128.h>
+
+#include <protocol/client_opcode.hxx>
+#include <document_id.hxx>
+
+namespace couchbase::protocol
+{
+
+class decrement_response_body
+{
+  public:
+    static const inline client_opcode opcode = client_opcode::decrement;
+
+  private:
+    mutation_token token_;
+    std::uint64_t content_;
+
+  public:
+    std::uint64_t content()
+    {
+        return content_;
+    }
+
+    mutation_token& token()
+    {
+        return token_;
+    }
+
+    bool parse(protocol::status status,
+               const header_buffer& header,
+               std::uint8_t framing_extras_size,
+               std::uint16_t /* key_size */,
+               std::uint8_t extras_size,
+               const std::vector<uint8_t>& body,
+               const cmd_info&)
+    {
+        Expects(header[1] == static_cast<uint8_t>(opcode));
+        if (status == protocol::status::success) {
+            using offset_type = std::vector<uint8_t>::difference_type;
+            offset_type offset = framing_extras_size;
+            if (extras_size == 16) {
+                memcpy(&token_.partition_uuid, body.data() + offset, sizeof(token_.partition_uuid));
+                token_.partition_uuid = utils::byte_swap_64(token_.partition_uuid);
+                offset += 8;
+
+                memcpy(&token_.sequence_number, body.data() + offset, sizeof(token_.sequence_number));
+                token_.sequence_number = utils::byte_swap_64(token_.sequence_number);
+                offset += 8;
+            }
+            memcpy(&content_, body.data() + offset, sizeof(content_));
+            content_ = utils::byte_swap_64(content_);
+            return true;
+        }
+        return false;
+    }
+};
+
+class decrement_request_body
+{
+  public:
+    using response_body_type = decrement_response_body;
+    static const inline client_opcode opcode = client_opcode::decrement;
+
+  private:
+    std::string key_;
+    std::vector<std::uint8_t> framing_extras_{};
+    std::uint64_t delta_{ 1 };
+    std::uint64_t initial_value_{ 0 };
+    std::uint32_t expiry_{ 0 };
+    std::vector<std::uint8_t> extras_{};
+
+  public:
+    void id(const document_id& id)
+    {
+        key_ = id.key;
+        if (id.collection_uid) {
+            unsigned_leb128<uint32_t> encoded(*id.collection_uid);
+            key_.insert(0, encoded.get());
+        }
+    }
+
+    void delta(std::uint64_t value)
+    {
+        delta_ = value;
+    }
+
+    void initial_value(std::uint64_t value)
+    {
+        initial_value_ = value;
+    }
+
+    void expiry(std::uint32_t value)
+    {
+        expiry_ = value;
+    }
+
+    void durability(protocol::durability_level level, std::optional<std::uint16_t> timeout)
+    {
+        if (level == protocol::durability_level::none) {
+            return;
+        }
+        auto frame_id = static_cast<uint8_t>(protocol::request_frame_info_id::durability_requirement);
+        if (timeout) {
+            framing_extras_.resize(4);
+            framing_extras_[0] = static_cast<std::uint8_t>((static_cast<std::uint32_t>(frame_id) << 4U) | 3U);
+            framing_extras_[1] = static_cast<std::uint8_t>(level);
+            uint16_t val = htons(*timeout);
+            memcpy(framing_extras_.data() + 2, &val, sizeof(val));
+        } else {
+            framing_extras_.resize(2);
+            framing_extras_[0] = static_cast<std::uint8_t>(static_cast<std::uint32_t>(frame_id) << 4U | 1U);
+            framing_extras_[1] = static_cast<std::uint8_t>(level);
+        }
+    }
+
+    const std::string& key()
+    {
+        return key_;
+    }
+
+    const std::vector<std::uint8_t>& framing_extras()
+    {
+        return framing_extras_;
+    }
+
+    const std::vector<std::uint8_t>& extras()
+    {
+        if (extras_.empty()) {
+            fill_extras();
+        }
+        return extras_;
+    }
+
+    const std::vector<std::uint8_t>& value()
+    {
+        static std::vector<std::uint8_t> empty;
+        return empty;
+    }
+
+    [[nodiscard]] std::size_t size()
+    {
+        if (extras_.empty()) {
+            fill_extras();
+        }
+        return extras_.size() + key_.size();
+    }
+
+  private:
+    void fill_extras()
+    {
+        extras_.resize(sizeof(delta_) + sizeof(initial_value_) + sizeof(expiry_));
+        using offset_type = std::vector<uint8_t>::difference_type;
+        offset_type offset = 0;
+
+        std::uint64_t num = utils::byte_swap_64(delta_);
+        memcpy(extras_.data() + offset, &num, sizeof(num));
+        offset += static_cast<offset_type>(sizeof(delta_));
+
+        num = utils::byte_swap_64(initial_value_);
+        memcpy(extras_.data() + offset, &num, sizeof(num));
+        offset += static_cast<offset_type>(sizeof(delta_));
+
+        std::uint32_t ttl = htonl(expiry_);
+        memcpy(extras_.data() + offset, &ttl, sizeof(ttl));
+    }
+};
+
+} // namespace couchbase::protocol