couchbase 3.0.1 → 3.0.2

data/ext/couchbase/diagnostics.hxx

@@ -90,7 +90,6 @@ struct endpoint_ping_info {
 struct ping_result {
     std::string id;
     std::string sdk;
-    std::uint64_t config_rev;
     std::map<service_type, std::vector<endpoint_ping_info>> services{};
 
     const int version{ 2 };
@@ -242,8 +241,6 @@ struct traits<couchbase::diag::ping_result> {
             { "version", r.version },
             { "id", r.id },
             { "sdk", r.sdk },
-            /* the highest known configuration revision */
-            { "config_rev", r.config_rev },
             { "services", services },
         };
     }

data/ext/couchbase/io/dns_client.hxx

@@ -109,7 +109,7 @@ class dns_client
                             resp.targets.emplace_back(
                               dns_srv_response::address{ fmt::format("{}", fmt::join(answer.target.labels, ".")), answer.port });
                         }
-                        return handler(resp);
+                        return handler(std::move(resp));
                     });
               });
             deadline_.expires_after(timeout);
@@ -179,7 +179,7 @@ class dns_client
                                                        resp.targets.emplace_back(dns_srv_response::address{
                                                          fmt::format("{}", fmt::join(answer.target.labels, ".")), answer.port });
                                                    }
-                                                   return handler(resp);
+                                                   return handler(std::move(resp));
                                                });
                                          });
                     });

data/ext/couchbase/io/http_command.hxx

@@ -41,8 +41,11 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
     template<typename Handler>
     void send_to(std::shared_ptr<io::http_session> session, Handler&& handler)
     {
-        encoded.type = Request::type;
-        request.encode_to(encoded, session->http_context());
+        encoded.type = request.type;
+        auto encoding_ec = request.encode_to(encoded, session->http_context());
+        if (encoding_ec) {
+            return handler(make_response(encoding_ec, request, {}));
+        }
         encoded.headers["client-context-id"] = request.client_context_id;
         auto log_prefix = session->log_prefix();
         spdlog::trace(R"({} HTTP request: {}, method={}, path="{}", client_context_id="{}", timeout={}ms)",
@@ -77,7 +80,7 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
                                                       resp.status_code,
                                                       spdlog::to_hex(resp.body));
                                          try {
-                                             handler(make_response(ec, self->request, resp));
+                                             handler(make_response(ec, self->request, std::move(msg)));
                                          } catch (priv::retry_http_request) {
                                              self->send_to(session, std::forward<Handler>(handler));
                                          }

data/ext/couchbase/io/http_session.hxx

@@ -60,14 +60,7 @@ class http_session : public std::enable_shared_from_this<http_session>
       , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
-      , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
-                                BACKEND_VERSION_MAJOR,
-                                BACKEND_VERSION_MINOR,
-                                BACKEND_VERSION_PATCH,
-                                BACKEND_GIT_REVISION,
-                                client_id_,
-                                id_,
-                                BACKEND_SYSTEM))
+      , user_agent_(fmt::format("{}; client/{}; session/{}; {}", couchbase::sdk_id(), client_id_, id_, BACKEND_SYSTEM))
       , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
       , http_ctx_(std::move(http_ctx))
     {
@@ -92,14 +85,7 @@ class http_session : public std::enable_shared_from_this<http_session>
       , credentials_(credentials)
       , hostname_(hostname)
       , service_(service)
-      , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
-                                BACKEND_VERSION_MAJOR,
-                                BACKEND_VERSION_MINOR,
-                                BACKEND_VERSION_PATCH,
-                                BACKEND_GIT_REVISION,
-                                client_id_,
-                                id_,
-                                BACKEND_SYSTEM))
+      , user_agent_(fmt::format("{}; client/{}; session/{}; {}", couchbase::sdk_id(), client_id_, id_, BACKEND_SYSTEM))
       , log_prefix_(fmt::format("[{}/{}]", client_id_, id_))
       , http_ctx_(std::move(http_ctx))
     {
@@ -115,6 +101,16 @@ class http_session : public std::enable_shared_from_this<http_session>
         return http_ctx_;
     }
 
+    std::string remote_address() const
+    {
+        return fmt::format("{}:{}", endpoint_address_, endpoint_.port());
+    }
+
+    std::string local_address() const
+    {
+        return fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port());
+    }
+
     [[nodiscard]] diag::endpoint_diag_info diag_info() const
     {
         return { type_,
@@ -122,8 +118,8 @@ class http_session : public std::enable_shared_from_this<http_session>
                  last_active_.time_since_epoch().count() == 0 ? std::nullopt
                                                               : std::make_optional(std::chrono::duration_cast<std::chrono::microseconds>(
                                                                   std::chrono::steady_clock::now() - last_active_)),
-                 fmt::format("{}:{}", endpoint_address_, endpoint_.port()),
-                 fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port()),
+                 remote_address(),
+                 local_address(),
                  state_ };
     }
 

data/ext/couchbase/io/http_session_manager.hxx

@@ -20,6 +20,8 @@
 #include <io/http_session.hxx>
 #include <service_type.hxx>
 #include <io/http_context.hxx>
+#include <operations/http_noop.hxx>
+#include <io/http_command.hxx>
 
 #include <random>
 
@@ -55,12 +57,91 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
 
         for (const auto& list : busy_sessions_) {
             for (const auto& session : list.second) {
-                res.services[list.first].emplace_back(session->diag_info());
+                if (session) {
+                    res.services[list.first].emplace_back(session->diag_info());
+                }
             }
         }
         for (const auto& list : idle_sessions_) {
             for (const auto& session : list.second) {
-                res.services[list.first].emplace_back(session->diag_info());
+                if (session) {
+                    res.services[list.first].emplace_back(session->diag_info());
+                }
+            }
+        }
+    }
+
+    template<typename Collector>
+    void ping(std::set<service_type> services, std::shared_ptr<Collector> collector, const couchbase::cluster_credentials& credentials)
+    {
+        std::array<service_type, 4> known_types{ service_type::query, service_type::analytics, service_type::search, service_type::views };
+        for (auto& node : config_.nodes) {
+            for (auto type : known_types) {
+                if (services.find(type) == services.end()) {
+                    continue;
+                }
+                std::uint16_t port = 0;
+                port = node.port_or(options_.network, type, options_.enable_tls, 0);
+                if (port != 0) {
+                    std::scoped_lock lock(sessions_mutex_);
+                    std::shared_ptr<http_session> session;
+                    session = options_.enable_tls ? std::make_shared<http_session>(type,
+                                                                                   client_id_,
+                                                                                   ctx_,
+                                                                                   tls_,
+                                                                                   credentials,
+                                                                                   node.hostname_for(options_.network),
+                                                                                   std::to_string(port),
+                                                                                   http_context{ config_, options_, query_cache_ })
+                                                  : std::make_shared<http_session>(type,
+                                                                                   client_id_,
+                                                                                   ctx_,
+                                                                                   credentials,
+                                                                                   node.hostname_for(options_.network),
+                                                                                   std::to_string(port),
+                                                                                   http_context{ config_, options_, query_cache_ });
+                    session->start();
+                    session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {
+                        for (auto& s : self->busy_sessions_[type]) {
+                            if (s && s->id() == id) {
+                                s.reset();
+                            }
+                        }
+                        for (auto& s : self->idle_sessions_[type]) {
+                            if (s && s->id() == id) {
+                                s.reset();
+                            }
+                        }
+                    });
+                    busy_sessions_[type].push_back(session);
+                    operations::http_noop_request request{};
+                    request.type = type;
+                    auto cmd = std::make_shared<operations::http_command<operations::http_noop_request>>(ctx_, request);
+                    cmd->send_to(session,
+                                 [start = std::chrono::steady_clock::now(),
+                                  self = shared_from_this(),
+                                  type,
+                                  session,
+                                  handler = collector->build_reporter()](operations::http_noop_response&& resp) mutable {
+                                     diag::ping_state state = diag::ping_state::ok;
+                                     std::optional<std::string> error{};
+                                     if (resp.ec) {
+                                         state = diag::ping_state::error;
+                                         error.emplace(fmt::format(
+                                           "code={}, message={}, http_code={}", resp.ec.value(), resp.ec.message(), resp.status_code));
+                                     }
+                                     handler(diag::endpoint_ping_info{
+                                       type,
+                                       session->id(),
+                                       std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start),
+                                       session->remote_address(),
+                                       session->local_address(),
+                                       state,
+                                       {},
+                                       error });
+                                     self->check_in(type, session);
+                                 });
+                }
             }
         }
     }

data/ext/couchbase/io/mcbp_command.hxx

@@ -167,7 +167,10 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager,
                 }
             }
         }
-        request.encode_to(encoded);
+        auto encoding_ec = request.encode_to(encoded, session_->context());
+        if (encoding_ec) {
+            return invoke_handler(encoding_ec);
+        }
 
         session_->write_and_subscribe(
           request.opaque,

data/ext/couchbase/io/mcbp_context.hxx

@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <vector>
+#include <configuration.hxx>
+#include <protocol/hello_feature.hxx>
+
+namespace couchbase
+{
+
+struct mcbp_context {
+    const std::optional<configuration>& config;
+    const std::vector<protocol::hello_feature>& supported_features;
+
+    [[nodiscard]] bool supports_feature(protocol::hello_feature feature)
+    {
+        return std::find(supported_features.begin(), supported_features.end(), feature) != supported_features.end();
+    }
+};
+
+} // namespace couchbase

data/ext/couchbase/io/mcbp_session.hxx

@@ -29,6 +29,7 @@
 #include <io/mcbp_parser.hxx>
 #include <io/streams.hxx>
 #include <io/retry_orchestrator.hxx>
+#include <io/mcbp_context.hxx>
 
 #include <timeout_defaults.hxx>
 
@@ -36,6 +37,7 @@
 #include <protocol/client_request.hxx>
 #include <protocol/client_response.hxx>
 #include <protocol/server_request.hxx>
+#include <protocol/cmd_noop.hxx>
 #include <protocol/cmd_hello.hxx>
 #include <protocol/cmd_sasl_list_mechs.hxx>
 #include <protocol/cmd_sasl_auth.hxx>
@@ -127,10 +129,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                   { "SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1", "PLAIN" })
         {
             tao::json::value user_agent{
-                { "a",
-                  fmt::format(
-                    "ruby/{}.{}.{}/{}", BACKEND_VERSION_MAJOR, BACKEND_VERSION_MINOR, BACKEND_VERSION_PATCH, BACKEND_GIT_REVISION) },
-                { "i", fmt::format("{}/{}", session_->client_id_, session_->id_) }
+                { "a", couchbase::sdk_id() },
+                { "i", fmt::format("{}/{}", session_->client_id_, session_->id_) },
             };
             protocol::client_request<protocol::hello_request_body> hello_req;
             hello_req.opaque(session_->next_opaque());
@@ -387,6 +387,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                                              resp.opaque());
                             }
                         } break;
+                        case protocol::client_opcode::noop:
                         case protocol::client_opcode::get_collections_manifest:
                         case protocol::client_opcode::get_collection_id:
                         case protocol::client_opcode::get:
@@ -396,6 +397,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         case protocol::client_opcode::insert:
                         case protocol::client_opcode::replace:
                         case protocol::client_opcode::upsert:
+                        case protocol::client_opcode::append:
+                        case protocol::client_opcode::prepend:
                         case protocol::client_opcode::remove:
                         case protocol::client_opcode::observe:
                         case protocol::client_opcode::unlock:
@@ -405,8 +408,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                         case protocol::client_opcode::subdoc_multi_mutation: {
                             std::uint32_t opaque = msg.header.opaque;
                             std::uint16_t status = ntohs(msg.header.specific);
+                            session_->command_handlers_mutex_.lock();
                             auto handler = session_->command_handlers_.find(opaque);
-                            if (handler != session_->command_handlers_.end()) {
+                            if (handler != session_->command_handlers_.end() && handler->second) {
                                 auto ec = session_->map_status_code(opcode, status);
                                 spdlog::trace("{} MCBP invoke operation handler: opcode={}, opaque={}, status={}, ec={}",
                                               session_->log_prefix_,
@@ -414,10 +418,12 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                                               opaque,
                                               protocol::status_to_string(status),
                                               ec.message());
-                                auto fun = handler->second;
+                                auto fun = std::move(handler->second);
                                 session_->command_handlers_.erase(handler);
+                                session_->command_handlers_mutex_.unlock();
                                 fun(ec, retry_reason::do_not_retry, std::move(msg));
                             } else {
+                                session_->command_handlers_mutex_.unlock();
                                 spdlog::debug("{} unexpected orphan response: opcode={}, opaque={}, status={}",
                                               session_->log_prefix_,
                                               opcode,
@@ -537,6 +543,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         return log_prefix_;
     }
 
+    std::string remote_address() const
+    {
+        return fmt::format("{}:{}", endpoint_address_, endpoint_.port());
+    }
+
+    std::string local_address() const
+    {
+        return fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port());
+    }
+
     [[nodiscard]] diag::endpoint_diag_info diag_info() const
     {
         return { service_type::kv,
@@ -544,12 +560,44 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
                  last_active_.time_since_epoch().count() == 0 ? std::nullopt
                                                               : std::make_optional(std::chrono::duration_cast<std::chrono::microseconds>(
                                                                   std::chrono::steady_clock::now() - last_active_)),
-                 fmt::format("{}:{}", endpoint_address_, endpoint_.port()),
-                 fmt::format("{}:{}", local_endpoint_address_, local_endpoint_.port()),
+                 remote_address(),
+                 local_address(),
                  state_,
                  bucket_name_ };
     }
 
+    template<typename Handler>
+    void ping(Handler&& handler)
+    {
+        protocol::client_request<protocol::mcbp_noop_request_body> req;
+        req.opaque(next_opaque());
+        write_and_subscribe(req.opaque(),
+                            req.data(false),
+                            [start = std::chrono::steady_clock::now(), self = shared_from_this(), handler](
+                              std::error_code ec, retry_reason reason, io::mcbp_message&& /* msg */) {
+                                diag::ping_state state = diag::ping_state::ok;
+                                std::optional<std::string> error{};
+                                if (ec) {
+                                    state = diag::ping_state::error;
+                                    error.emplace(fmt::format("code={}, message={}, reason={}", ec.value(), ec.message(), reason));
+                                }
+                                handler(diag::endpoint_ping_info{
+                                  service_type::kv,
+                                  self->id_,
+                                  std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start),
+                                  self->remote_address(),
+                                  self->local_address(),
+                                  state,
+                                  self->bucket_name_,
+                                  error });
+                            });
+    }
+
+    [[nodiscard]] mcbp_context context() const
+    {
+        return { config_, supported_features_ };
+    }
+
     void bootstrap(std::function<void(std::error_code, configuration)>&& handler, bool retry_on_bucket_not_found = false)
     {
         retry_bootstrap_on_bucket_not_found_ = retry_on_bucket_not_found;
@@ -648,11 +696,18 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (handler_) {
             handler_->stop();
         }
-        for (auto& handler : command_handlers_) {
-            spdlog::debug("{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
-            handler.second(ec, reason, {});
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            for (auto& handler : command_handlers_) {
+                if (handler.second) {
+                    spdlog::debug(
+                      "{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
+                    auto fun = std::move(handler.second);
+                    fun(ec, reason, {});
+                }
+            }
+            command_handlers_.clear();
         }
-        command_handlers_.clear();
         config_listeners_.clear();
         if (on_stop_handler_) {
             on_stop_handler_(reason);
@@ -700,7 +755,10 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             handler(std::make_error_code(error::common_errc::request_canceled), retry_reason::socket_closed_while_in_flight, {});
             return;
         }
-        command_handlers_.emplace(opaque, std::move(handler));
+        {
+            std::scoped_lock lock(command_handlers_mutex_);
+            command_handlers_.emplace(opaque, std::move(handler));
+        }
         if (bootstrapped_ && stream_->is_open()) {
             write_and_flush(data);
         } else {
@@ -715,13 +773,19 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
         if (stopped_) {
             return false;
         }
+        command_handlers_mutex_.lock();
         auto handler = command_handlers_.find(opaque);
         if (handler != command_handlers_.end()) {
             spdlog::debug("{} MCBP cancel operation, opaque={}, ec={} ({})", log_prefix_, opaque, ec.value(), ec.message());
-            handler->second(ec, reason, {});
-            command_handlers_.erase(handler);
-            return true;
+            if (handler->second) {
+                auto fun = std::move(handler->second);
+                command_handlers_.erase(handler);
+                command_handlers_mutex_.unlock();
+                fun(ec, reason, {});
+                return true;
+            }
         }
+        command_handlers_mutex_.unlock();
         return false;
     }
 
@@ -1142,22 +1206,19 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
               self->last_active_ = std::chrono::steady_clock::now();
               if (ec) {
                   if (stream_id != self->stream_->id()) {
-                      spdlog::error(
-                        R"({} ignore IO error while reading from the socket: {} ({}), pending_handlers={}, old_id="{}", new_id="{}")",
-                        self->log_prefix_,
-                        ec.value(),
-                        ec.message(),
-                        self->command_handlers_.size(),
-                        stream_id,
-                        self->stream_->id());
+                      spdlog::error(R"({} ignore IO error while reading from the socket: {} ({}), old_id="{}", new_id="{}")",
+                                    self->log_prefix_,
+                                    ec.value(),
+                                    ec.message(),
+                                    stream_id,
+                                    self->stream_->id());
                       return;
                   }
-                  spdlog::error(R"({} IO error while reading from the socket("{}"): {} ({}), pending_handlers={})",
+                  spdlog::error(R"({} IO error while reading from the socket("{}"): {} ({}))",
                                 self->log_prefix_,
                                 self->stream_->id(),
                                 ec.value(),
-                                ec.message(),
-                                self->command_handlers_.size());
+                                ec.message());
                   return self->stop(retry_reason::socket_closed_while_in_flight);
               }
               self->parser_.feed(self->input_buffer_.data(), self->input_buffer_.data() + ssize_t(bytes_transferred));
@@ -1212,12 +1273,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
             }
             self->last_active_ = std::chrono::steady_clock::now();
             if (ec) {
-                spdlog::error(R"({} IO error while writing to the socket("{}"): {} ({}), pending_handlers={})",
+                spdlog::error(R"({} IO error while writing to the socket("{}"): {} ({}))",
                               self->log_prefix_,
                               self->stream_->id(),
                               ec.value(),
-                              ec.message(),
-                              self->command_handlers_.size());
+                              ec.message());
                 return self->stop(retry_reason::socket_closed_while_in_flight);
             }
             {
@@ -1242,6 +1302,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
     mcbp_parser parser_;
     std::unique_ptr<message_handler> handler_;
     std::function<void(std::error_code, const configuration&)> bootstrap_handler_{};
+    std::mutex command_handlers_mutex_{};
     std::map<uint32_t, std::function<void(std::error_code, retry_reason, io::mcbp_message&&)>> command_handlers_{};
     std::vector<std::function<void(const configuration&)>> config_listeners_{};
     std::function<void(io::retry_reason)> on_stop_handler_{};