couchbase 3.0.0.alpha.2 → 3.0.0.alpha.3

data/ext/couchbase/io/http_session.hxx

@@ -33,6 +33,7 @@
 #include <io/http_parser.hxx>
 #include <io/http_message.hxx>
 #include <platform/base64.h>
+#include <timeout_defaults.hxx>
 
 namespace couchbase::io
 {
@@ -73,9 +74,14 @@ class http_session : public std::enable_shared_from_this
         stop();
     }
 
-    [[nodiscard]] std::string id()
+    [[nodiscard]] uuid::uuid_t id()
     {
-        return uuid::to_string(id_);
+        return id_;
+    }
+
+    void on_stop(std::function<void()> handler)
+    {
+        on_stop_handler_ = std::move(handler);
     }
 
     void stop()
@@ -85,6 +91,16 @@ class http_session : public std::enable_shared_from_this
             socket_.close();
         }
         deadline_timer_.cancel();
+
+        for (auto& handler : command_handlers_) {
+            handler(std::make_error_code(error::common_errc::ambiguous_timeout), {});
+        }
+        command_handlers_.clear();
+
+        if (on_stop_handler_) {
+            on_stop_handler_();
+            on_stop_handler_ = nullptr;
+        }
     }
 
     bool is_stopped()
@@ -105,7 +121,7 @@ class http_session : public std::enable_shared_from_this
         if (stopped_) {
             return;
         }
-        output_buffer_.emplace_back(buf.begin(), buf.end());
+        output_buffer_.emplace_back(std::vector<uint8_t>{ buf.begin(), buf.end() });
     }
 
     void flush()
@@ -155,7 +171,7 @@ class http_session : public std::enable_shared_from_this
     {
         if (it != endpoints_.end()) {
             spdlog::trace("connecting to {}:{}", it->endpoint().address().to_string(), it->endpoint().port());
-            deadline_timer_.expires_after(std::chrono::seconds(10));
+            deadline_timer_.expires_after(timeout_defaults::connect_timeout);
             socket_.async_connect(it->endpoint(), std::bind(&http_session::on_connect, this, std::placeholders::_1, it));
         } else {
             spdlog::error("no more endpoints left to connect");
@@ -213,9 +229,11 @@ class http_session : public std::enable_shared_from_this
               switch (self->parser_.feed(reinterpret_cast<const char*>(self->input_buffer_.data()), bytes_transferred)) {
                   case http_parser::status::ok:
                       if (self->parser_.complete) {
-                          auto handler = self->command_handlers_.front();
-                          self->command_handlers_.pop_front();
-                          handler({}, std::move(self->parser_.response));
+                          if (!self->command_handlers_.empty()) {
+                              auto handler = self->command_handlers_.front();
+                              self->command_handlers_.pop_front();
+                              handler({}, std::move(self->parser_.response));
+                          }
                           self->parser_.reset();
                           return;
                       }
@@ -273,6 +291,8 @@ class http_session : public std::enable_shared_from_this
     bool stopped_{ false };
     bool connected_{ false };
 
+    std::function<void()> on_stop_handler_{ nullptr };
+
     std::list<std::function<void(std::error_code, io::http_response&&)>> command_handlers_{};
     http_parser parser_{};
     std::array<std::uint8_t, 16384> input_buffer_{};

data/ext/couchbase/io/mcbp_parser.hxx

@@ -73,6 +73,8 @@ struct mcbp_parser {
             if (success) {
                 std::copy(uncompressed.begin(), uncompressed.end(), std::back_inserter(msg.body));
                 use_raw_value = false;
+                // patch header with new body size
+                msg.header.bodylen = htonl(static_cast<std::uint32_t>(prefix_size + uncompressed.size()));
             }
         }
         if (use_raw_value) {

data/ext/couchbase/io/mcbp_session.hxx

@@ -28,6 +28,8 @@
 #include <io/mcbp_message.hxx>
 #include <io/mcbp_parser.hxx>
 
+#include <timeout_defaults.hxx>
+
 #include <protocol/hello_feature.hxx>
 #include <protocol/client_request.hxx>
 #include <protocol/client_response.hxx>
@@ -39,7 +41,6 @@
 #include <protocol/cmd_select_bucket.hxx>
 #include <protocol/cmd_get_cluster_config.hxx>
 #include <protocol/cmd_get_error_map.hxx>
-#include <protocol/cmd_get_collections_manifest.hxx>
 #include <protocol/cmd_get.hxx>
 #include <protocol/cmd_cluster_map_change_notification.hxx>
 
@@ -55,6 +56,35 @@ namespace couchbase::io
 
 class mcbp_session : public std::enable_shared_from_this<mcbp_session>
 {
+    class collection_cache
+    {
+      private:
+        std::map<std::string, std::uint32_t> cid_map_{ { "_default._default", 0 } };
+
+      public:
+        [[nodiscard]] std::optional<std::uint32_t> get(const std::string& path)
+        {
+            Expects(!path.empty());
+            auto ptr = cid_map_.find(path);
+            if (ptr != cid_map_.end()) {
+                return ptr->second;
+            }
+            return {};
+        }
+
+        void update(const std::string& path, std::uint32_t id)
+        {
+            Expects(!path.empty());
+            cid_map_[path] = id;
+        }
+
+        void reset()
+        {
+            cid_map_.clear();
+            cid_map_["_default._default"] = 0;
+        }
+    };
+
     class message_handler
     {
       public:
@@ -138,10 +168,6 @@ class mcbp_session : public std::enable_shared_from_this
                 sb_req.opaque(session_->next_opaque());
                 sb_req.body().bucket_name(session_->bucket_name_.value());
                 session_->write(sb_req.data());
-
-                protocol::client_request<protocol::get_collections_manifest_request_body> gcm_req;
-                gcm_req.opaque(session_->next_opaque());
-                session_->write(gcm_req.data());
             }
             protocol::client_request<protocol::get_cluster_config_request_body> cfg_req;
             cfg_req.opaque(session_->next_opaque());
@@ -216,21 +242,6 @@ class mcbp_session : public std::enable_shared_from_this
                         return complete(std::make_error_code(error::network_errc::protocol_error));
                     }
                 } break;
-                case protocol::client_opcode::get_collections_manifest: {
-                    protocol::client_response<protocol::get_collections_manifest_response_body> resp(msg);
-                    if (resp.status() == protocol::status::success) {
-                        session_->manifest_.emplace(resp.body().manifest());
-                        spdlog::trace(
-                          "collections manifest for bucket \"{}\": {}", session_->bucket_name_.value_or(""), *session_->manifest_);
-                    } else if (resp.status() == protocol::status::no_collections_manifest) {
-                        spdlog::trace("collection manifest is not available for bucket \"{}\": {}",
-                                      session_->bucket_name_.value_or(""),
-                                      resp.error_message());
-                    } else {
-                        spdlog::warn("unexpected message status during bootstrap: {} (opcode={})", resp.error_message(), opcode);
-                        return complete(std::make_error_code(error::network_errc::protocol_error));
-                    }
-                } break;
                 case protocol::client_opcode::select_bucket: {
                     protocol::client_response<protocol::select_bucket_response_body> resp(msg);
                     if (resp.status() == protocol::status::success) {
@@ -320,6 +331,7 @@ class mcbp_session : public std::enable_shared_from_this
                                 spdlog::warn("unexpected message status: {}", resp.error_message());
                             }
                         } break;
+                        case protocol::client_opcode::get_collection_id:
                         case protocol::client_opcode::get:
                         case protocol::client_opcode::get_and_lock:
                         case protocol::client_opcode::get_and_touch:
@@ -481,9 +493,16 @@ class mcbp_session : public std::enable_shared_from_this
         }
     }
 
-    [[nodiscard]] std::optional<collections_manifest> manifest()
+    void cancel(uint32_t opaque, std::error_code ec)
     {
-        return manifest_;
+        if (stopped_) {
+            return;
+        }
+        auto handler = command_handlers_.find(opaque);
+        if (handler != command_handlers_.end()) {
+            handler->second(ec, {});
+            command_handlers_.erase(handler);
+        }
     }
 
     bool supports_feature(protocol::hello_feature feature)
@@ -663,6 +682,16 @@ class mcbp_session : public std::enable_shared_from_this
         }
     }
 
+    std::optional<std::uint32_t> get_collection_uid(const std::string& collection_path)
+    {
+        return collection_cache_.get(collection_path);
+    }
+
+    void update_collection_uid(const std::string& path, std::uint32_t uid)
+    {
+        collection_cache_.update(path, uid);
+    }
+
   private:
     void invoke_bootstrap_handler(std::error_code ec)
     {
@@ -704,7 +733,7 @@ class mcbp_session : public std::enable_shared_from_this
         }
         if (it != endpoints_.end()) {
             spdlog::trace("connecting to {}:{}", it->endpoint().address().to_string(), it->endpoint().port());
-            deadline_timer_.expires_after(std::chrono::seconds(10));
+            deadline_timer_.expires_after(timeout_defaults::connect_timeout);
             socket_.async_connect(it->endpoint(), std::bind(&mcbp_session::on_connect, this, std::placeholders::_1, it));
         } else {
             spdlog::error("no more endpoints left to connect");
@@ -852,7 +881,7 @@ class mcbp_session : public std::enable_shared_from_this
     std::vector<protocol::hello_feature> supported_features_;
     std::optional<configuration> config_;
     std::optional<error_map> errmap_;
-    std::optional<collections_manifest> manifest_;
+    collection_cache collection_cache_;
 
     std::atomic_bool reading_{ false };
 };

data/ext/couchbase/io/session_manager.hxx

@@ -25,7 +25,7 @@
 namespace couchbase::io
 {
 
-class session_manager
+class session_manager : public std::enable_shared_from_this<session_manager>
 {
   public:
     session_manager(uuid::uuid_t client_id, asio::io_context& ctx)
@@ -57,11 +57,16 @@ class session_manager
             }
             config_.nodes.size();
             auto session = std::make_shared<http_session>(client_id_, ctx_, username, password, hostname, std::to_string(port));
+            session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {
+                self->busy_sessions_[type].remove_if([id](const auto& s) -> bool { return s->id() == id; });
+                self->idle_sessions_[type].remove_if([id](const auto& s) -> bool { return s->id() == id; });
+            });
             busy_sessions_[type].push_back(session);
             return session;
         }
         auto session = idle_sessions_[type].front();
         idle_sessions_[type].pop_front();
+        busy_sessions_[type].push_back(session);
         return session;
     }
 

data/ext/couchbase/operations.hxx

@@ -18,6 +18,7 @@
 #pragma once
 
 #include <document_id.hxx>
+#include <timeout_defaults.hxx>
 
 #include <operations/document_get.hxx>
 #include <operations/document_get_and_lock.hxx>
@@ -33,8 +34,10 @@
 #include <operations/document_unlock.hxx>
 #include <operations/document_increment.hxx>
 #include <operations/document_decrement.hxx>
+#include <operations/document_get_projected.hxx>
 
 #include <operations/document_query.hxx>
+#include <operations/document_search.hxx>
 
 #include <operations/bucket_get_all.hxx>
 #include <operations/bucket_get.hxx>
@@ -56,4 +59,14 @@
 #include <operations/query_index_create.hxx>
 #include <operations/query_index_build_deferred.hxx>
 
+#include <operations/search_index_get_all.hxx>
+#include <operations/search_index_get.hxx>
+#include <operations/search_index_get_documents_count.hxx>
+#include <operations/search_index_upsert.hxx>
+#include <operations/search_index_drop.hxx>
+#include <operations/search_index_control_ingest.hxx>
+#include <operations/search_index_control_query.hxx>
+#include <operations/search_index_control_plan_freeze.hxx>
+#include <operations/search_index_analyze_document.hxx>
+
 #include <operations/command.hxx>

data/ext/couchbase/operations/bucket_create.hxx

@@ -39,6 +39,7 @@ struct bucket_create_request {
     static const inline service_type type = service_type::management;
 
     bucket_settings bucket{};
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/bucket_drop.hxx

@@ -34,6 +34,7 @@ struct bucket_drop_request {
     static const inline service_type type = service_type::management;
 
     std::string name;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/bucket_flush.hxx

@@ -34,6 +34,7 @@ struct bucket_flush_request {
     static const inline service_type type = service_type::management;
 
     std::string name;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/bucket_get.hxx

@@ -38,6 +38,7 @@ struct bucket_get_request {
     static const inline service_type type = service_type::management;
 
     std::string name;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/bucket_get_all.hxx

@@ -36,6 +36,7 @@ struct bucket_get_all_request {
     using encoded_response_type = io::http_response;
 
     static const inline service_type type = service_type::management;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/bucket_update.hxx

@@ -38,6 +38,7 @@ struct bucket_update_request {
     using encoded_response_type = io::http_response;
 
     static const inline service_type type = service_type::management;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     bucket_settings bucket{};
 

data/ext/couchbase/operations/cluster_developer_preview_enable.hxx

@@ -35,6 +35,7 @@ struct cluster_developer_preview_enable_request {
     using encoded_response_type = io::http_response;
 
     static const inline service_type type = service_type::management;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/collection_create.hxx

@@ -42,6 +42,7 @@ struct collection_create_request {
     std::string scope_name;
     std::string collection_name;
     std::uint32_t max_expiry{ 0 };
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {
@@ -62,7 +63,11 @@ make_response(std::error_code ec, collection_create_request&, collection_create_
     if (!ec) {
         switch (encoded.status_code) {
             case 400:
-                response.ec = std::make_error_code(error::management_errc::collection_exists);
+                if (encoded.body.find("Collection with this name already exists") != std::string::npos) {
+                    response.ec = std::make_error_code(error::management_errc::collection_exists);
+                } else {
+                    response.ec = std::make_error_code(error::common_errc::invalid_argument);
+                }
                 break;
             case 404:
                 if (encoded.body.find("Scope with this name is not found") != std::string::npos) {

data/ext/couchbase/operations/collection_drop.hxx

@@ -41,6 +41,7 @@ struct collection_drop_request {
     std::string bucket_name;
     std::string scope_name;
     std::string collection_name;
+    std::chrono::milliseconds timeout{ timeout_defaults::management_timeout };
 
     void encode_to(encoded_request_type& encoded)
     {

data/ext/couchbase/operations/command.hxx

@@ -19,6 +19,7 @@
 
 #include <io/mcbp_session.hxx>
 #include <io/http_session.hxx>
+#include <protocol/cmd_get_collection_id.hxx>
 
 namespace couchbase::operations
 {
@@ -28,30 +29,103 @@ struct command : public std::enable_shared_from_this> {
     using encoded_request_type = typename Request::encoded_request_type;
     using encoded_response_type = typename Request::encoded_response_type;
     asio::steady_timer deadline;
+    asio::steady_timer retry_backoff;
     Request request;
     encoded_request_type encoded;
 
     command(asio::io_context& ctx, Request&& req)
       : deadline(ctx)
+      , retry_backoff(ctx)
       , request(req)
     {
     }
 
+    template<typename Handler>
+    void request_collection_id(std::shared_ptr<io::mcbp_session> session, Handler&& handler)
+    {
+        protocol::client_request<protocol::get_collection_id_request_body> req;
+        req.opaque(session->next_opaque());
+        req.body().collection_path(request.id.collection);
+        session->write_and_subscribe(req.opaque(),
+                                     req.data(session->supports_feature(protocol::hello_feature::snappy)),
+                                     [self = this->shared_from_this(), session, handler = std::forward<Handler>(handler)](
+                                       std::error_code ec, io::mcbp_message&& msg) mutable {
+                                         if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
+                                             if (self->request.id.collection_uid) {
+                                                 return self->handle_unknown_collection(session, std::forward<Handler>(handler));
+                                             }
+                                             return handler(make_response(ec, self->request, {}));
+                                         }
+                                         if (ec) {
+                                             return handler(make_response(ec, self->request, {}));
+                                         }
+                                         protocol::client_response<protocol::get_collection_id_response_body> resp(msg);
+                                         session->update_collection_uid(self->request.id.collection, resp.body().collection_uid());
+                                         self->request.id.collection_uid = resp.body().collection_uid();
+                                         return self->send_to(session, std::forward<Handler>(handler));
+                                     });
+    }
+
+    template<typename Handler>
+    void handle_unknown_collection(std::shared_ptr<io::mcbp_session> session, Handler&& handler)
+    {
+        auto backoff = std::chrono::milliseconds(500);
+        if (std::chrono::steady_clock::now() + backoff > deadline.expiry()) {
+            return handler(make_response(std::make_error_code(error::common_errc::ambiguous_timeout), request, {}));
+        }
+        retry_backoff.expires_after(backoff);
+        retry_backoff.async_wait(
+          [self = this->shared_from_this(), session, handler = std::forward<Handler>(handler)](std::error_code ec) mutable {
+              if (ec == asio::error::operation_aborted) {
+                  return handler(make_response(std::make_error_code(error::common_errc::ambiguous_timeout), self->request, {}));
+              }
+              self->request_collection_id(session, std::forward<Handler>(handler));
+          });
+    }
+
     template<typename Handler>
     void send_to(std::shared_ptr<io::mcbp_session> session, Handler&& handler)
     {
-        request.opaque = session->next_opaque();
+        auto opaque = session->next_opaque();
+        request.opaque = opaque;
+        if (!request.id.collection_uid) {
+            if (session->supports_feature(protocol::hello_feature::collections)) {
+                auto collection_id = session->get_collection_uid(request.id.collection);
+                if (collection_id) {
+                    request.id.collection_uid = *collection_id;
+                } else {
+                    return request_collection_id(session, std::forward<Handler>(handler));
+                }
+            } else {
+                if (!request.id.collection.empty() && request.id.collection != "_default._default") {
+                    return handler(make_response(std::make_error_code(error::common_errc::unsupported_operation), request, {}));
+                }
+            }
+        }
         request.encode_to(encoded);
         session->write_and_subscribe(
           request.opaque,
           encoded.data(session->supports_feature(protocol::hello_feature::snappy)),
-          [self = this->shared_from_this(), handler = std::forward<Handler>(handler)](std::error_code ec, io::mcbp_message&& msg) mutable {
-              encoded_response_type resp(msg);
+          [self = this->shared_from_this(), session, handler = std::forward<Handler>(handler)](std::error_code ec,
+                                                                                               io::mcbp_message&& msg) mutable {
+              if (ec == asio::error::operation_aborted) {
+                  return handler(make_response(std::make_error_code(error::common_errc::ambiguous_timeout), self->request, {}));
+              }
               self->deadline.cancel();
+              self->retry_backoff.cancel();
+              encoded_response_type resp(msg);
+              if (resp.status() == protocol::status::unknown_collection) {
+                  return self->handle_unknown_collection(session, std::forward<Handler>(handler));
+              }
               handler(make_response(ec, self->request, resp));
           });
-        deadline.expires_after(std::chrono::milliseconds(2500));
-        deadline.async_wait(std::bind(&command<Request>::deadline_handler, this));
+        deadline.expires_after(request.timeout);
+        deadline.async_wait([session, opaque](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            session->cancel(opaque, asio::error::operation_aborted);
+        });
     }
 
     template<typename Handler>
@@ -65,12 +139,13 @@ struct command : public std::enable_shared_from_this> {
               self->deadline.cancel();
               handler(make_response(ec, self->request, resp));
           });
-        deadline.expires_after(std::chrono::milliseconds(2500));
-        deadline.async_wait(std::bind(&command<Request>::deadline_handler, this));
-    }
-
-    void deadline_handler()
-    {
+        deadline.expires_after(request.timeout);
+        deadline.async_wait([session](std::error_code ec) {
+            if (ec == asio::error::operation_aborted) {
+                return;
+            }
+            session->stop();
+        });
     }
 };