couchbase 3.8.1 → 3.9.0

data/ext/couchbase/core/io/streams.cxx

@@ -18,17 +18,247 @@
 #include "streams.hxx"
 
 #include "core/platform/uuid.h"
+#include "core/utils/process_id.hxx"
 
 #include <asio.hpp>
 #include <asio/error.hpp>
 #include <asio/ssl.hpp>
+#include <asio/ssl/host_name_verification.hpp>
+// ERR_get_error()/SSL_* come from the OpenSSL or BoringSSL headers pulled in by
+// <asio/ssl.hpp>; do not include <openssl/*.h> directly so the static BoringSSL
+// build (which ships its own headers) keeps working, matching core/io/mcbp_session.cxx.
+
+#include <algorithm>
+#include <cerrno>
+#include <memory>
+#include <mutex>
+
+#if !defined(_WIN32)
+#include <unistd.h>
+#endif
 
 namespace couchbase::core::io
 {
+namespace
+{
+/**
+ * Release a socket this process inherited across fork(2), without disturbing the
+ * connection the parent is still using.
+ *
+ * Two things differ from the ordinary close path, and both matter:
+ *
+ *  * No shutdown(2). It acts on the open file description, which fork(2) shares,
+ *    so it would send FIN and break the parent's connection. close(2) drops only
+ *    our own descriptor; the peer sees nothing.
+ *  * release() before closing, rather than asio's close(). asio's close() skips
+ *    EPOLL_CTL_DEL on the assumption that closing a descriptor removes its epoll
+ *    registration. Per epoll(7) that happens only once *every* descriptor
+ *    referring to the description is closed -- and the parent still holds one --
+ *    so the registration would outlive the descriptor_state asio recycles, which
+ *    is precisely the stale-pointer shape the reactor faults on. release() emits
+ *    the EPOLL_CTL_DEL explicitly and hands back the raw descriptor to close.
+ *
+ * release() also completes any outstanding asynchronous operations on this socket
+ * with operation_aborted, which is what we want: those reads belong to the parent.
+ */
+// Takes the basic_socket base rather than a concrete socket type, so the same code serves the
+// plain socket, a TLS stream's lowest_layer(), whose type is that base, and the DNS client's
+// sockets.
+template<typename Protocol>
+void
+detach_inherited_socket(asio::basic_socket<Protocol>& socket, asio::error_code& ec)
+{
+  if (!socket.is_open()) {
+    ec = asio::error::bad_descriptor;
+    return;
+  }
+
+  const auto handle = socket.release(ec);
+  if (ec) {
+    // Report it. Do not reach for asio's close() here: that is the very call
+    // whose epoll behaviour makes it unsafe on an inherited descriptor, and the
+    // descriptor goes away when this process exits anyway.
+    return;
+  }
+
+#if defined(_WIN32)
+  // Unreachable: without fork(2) no socket is ever inherited, so the owner pid
+  // always matches and close() never takes this path.
+  (void)handle;
+  ec = asio::error::operation_not_supported;
+#else
+  if (::close(handle) != 0) {
+    ec.assign(errno, asio::error::get_system_category());
+  }
+#endif
+}
+} // namespace
+
+auto
+configure_tls_handshake(asio::ssl::stream<asio::ip::tcp::socket>& stream,
+                        const std::string& hostname) -> std::error_code
+{
+  if (hostname.empty()) {
+    // Fail closed: a TLS client connection must have a hostname to verify the
+    // server's identity against. Returning success here would silently fall back
+    // to CA-only validation -- the CWE-297 problem this function exists to prevent.
+    return asio::error::make_error_code(asio::error::invalid_argument);
+  }
+
+  // Send SNI so that name-based / multi-tenant TLS endpoints present the correct
+  // certificate for this hostname.
+  {
+    // SSL_set_tlsext_host_name() is a macro that expands to an internal C-style
+    // cast to void*, which trips -Wold-style-cast; suppress it locally.
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#endif
+    ERR_clear_error();
+    const auto sni_result = SSL_set_tlsext_host_name(stream.native_handle(), hostname.c_str());
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+    if (sni_result != 1) {
+      // Prefer the underlying SSL reason (if any) so failures are diagnosable,
+      // and fall back to a generic error when the SSL error queue is empty.
+      if (const auto reason = ERR_get_error(); reason != 0) {
+        return { static_cast<int>(reason), asio::error::get_ssl_category() };
+      }
+      return asio::error::make_error_code(asio::error::invalid_argument);
+    }
+  }
+
+  // Verify that the presented certificate actually identifies `hostname`
+  // (SAN/CN). asio::ssl::verify_peer on its own only validates the certificate
+  // chain, not the name. In verify_none mode OpenSSL ignores the callback
+  // result, so this has no effect there.
+  std::error_code ec{};
+  stream.set_verify_callback(asio::ssl::host_name_verification(hostname), ec);
+  return ec;
+}
+
+// The sockets of one io_context, each with the process that opened it. Held weakly: a socket also
+// lives in the handlers that use it, such as a close posted before a fork, and is found there too.
+class inherited_stream_registry : public asio::execution_context::service
+{
+public:
+  static inline asio::execution_context::id id{};
+
+  explicit inherited_stream_registry(asio::execution_context& ctx)
+    : asio::execution_context::service(ctx)
+  {
+  }
+
+  void add(const std::shared_ptr<asio::ip::tcp::socket>& socket)
+  {
+    const std::scoped_lock lock(mutex_);
+    prune();
+    plain_.push_back({ socket, utils::current_process_id() });
+  }
+
+  void add(const std::shared_ptr<asio::ssl::stream<asio::ip::tcp::socket>>& stream)
+  {
+    const std::scoped_lock lock(mutex_);
+    prune();
+    tls_.push_back({ stream, utils::current_process_id() });
+  }
+
+  void add(const std::shared_ptr<asio::ip::udp::socket>& socket)
+  {
+    const std::scoped_lock lock(mutex_);
+    prune();
+    udp_.push_back({ socket, utils::current_process_id() });
+  }
+
+  // The detach result needs no handling. asio's reactive release() fails only on a socket that is
+  // not open, which is skipped here, and once it succeeds the socket is out of the reactor with its
+  // operations aborted, so a failed close(2) cannot let anything run on the parent's connection.
+  void detach_inherited()
+  {
+    const auto pid = utils::current_process_id();
+    const std::scoped_lock lock(mutex_);
+    for (const auto& [weak, owner_pid] : plain_) {
+      if (auto socket = weak.lock(); socket && owner_pid != pid && socket->is_open()) {
+        asio::error_code ec{};
+        detach_inherited_socket(*socket, ec);
+      }
+    }
+    for (const auto& [weak, owner_pid] : tls_) {
+      if (auto stream = weak.lock();
+          stream && owner_pid != pid && stream->lowest_layer().is_open()) {
+        asio::error_code ec{};
+        detach_inherited_socket(stream->lowest_layer(), ec);
+      }
+    }
+    for (const auto& [weak, owner_pid] : udp_) {
+      if (auto socket = weak.lock(); socket && owner_pid != pid && socket->is_open()) {
+        asio::error_code ec{};
+        detach_inherited_socket(*socket, ec);
+      }
+    }
+    prune();
+  }
+
+private:
+  template<typename Stream>
+  struct entry {
+    std::weak_ptr<Stream> stream;
+    std::int64_t owner_pid;
+  };
+
+  void shutdown() override
+  {
+  }
+
+  void prune()
+  {
+    const auto expired = [](const auto& e) {
+      return e.stream.expired();
+    };
+    plain_.erase(std::remove_if(plain_.begin(), plain_.end(), expired), plain_.end());
+    tls_.erase(std::remove_if(tls_.begin(), tls_.end(), expired), tls_.end());
+    udp_.erase(std::remove_if(udp_.begin(), udp_.end(), expired), udp_.end());
+  }
+
+  std::mutex mutex_{};
+  std::vector<entry<asio::ip::tcp::socket>> plain_{};
+  std::vector<entry<asio::ssl::stream<asio::ip::tcp::socket>>> tls_{};
+  std::vector<entry<asio::ip::udp::socket>> udp_{};
+};
+
+void
+detach_inherited_sockets(asio::io_context& ctx)
+{
+  if (asio::has_service<inherited_stream_registry>(ctx)) {
+    asio::use_service<inherited_stream_registry>(static_cast<asio::execution_context&>(ctx))
+      .detach_inherited();
+  }
+}
+
+void
+register_inherited_socket(asio::io_context& ctx,
+                          const std::shared_ptr<asio::ip::tcp::socket>& socket)
+{
+  asio::use_service<inherited_stream_registry>(static_cast<asio::execution_context&>(ctx))
+    .add(socket);
+}
+
+void
+register_inherited_socket(asio::io_context& ctx,
+                          const std::shared_ptr<asio::ip::udp::socket>& socket)
+{
+  asio::use_service<inherited_stream_registry>(static_cast<asio::execution_context&>(ctx))
+    .add(socket);
+}
+
 stream_impl::stream_impl(asio::io_context& ctx, bool is_tls)
-  : strand_(asio::make_strand(ctx))
+  : inherited_(
+      asio::use_service<inherited_stream_registry>(static_cast<asio::execution_context&>(ctx)))
+  , strand_(asio::make_strand(ctx))
   , tls_(is_tls)
   , id_(uuid::to_string(uuid::random()))
+  , owner_pid_(utils::current_process_id())
 {
 }
 
@@ -48,6 +278,7 @@ plain_stream_impl::plain_stream_impl(asio::io_context& ctx)
   : stream_impl(ctx, false)
   , stream_(std::make_shared<asio::ip::tcp::socket>(strand_))
 {
+  inherited_.add(stream_);
 }
 
 auto
@@ -79,12 +310,20 @@ plain_stream_impl::close(utils::movable_function<void(std::error_code)>&& handle
   if (!stream_) {
     return handler(asio::error::bad_descriptor);
   }
-  return asio::post(strand_, [stream = std::move(stream_), handler = std::move(handler)]() {
-    asio::error_code ec{};
-    stream->shutdown(asio::socket_base::shutdown_both, ec);
-    stream->close(ec);
-    handler(ec);
-  });
+  return asio::post(
+    strand_, [stream = std::move(stream_), owner_pid = owner_pid_, handler = std::move(handler)]() {
+      asio::error_code ec{};
+      // Decide here rather than when this was posted: io_.stop() at fork_prepare
+      // does not drain the queue, so a close queued before the fork runs in the
+      // child once the io_context is restarted, and there the answer differs.
+      if (owner_pid != utils::current_process_id()) {
+        detach_inherited_socket(*stream, ec);
+      } else {
+        stream->shutdown(asio::socket_base::shutdown_both, ec);
+        stream->close(ec);
+      }
+      handler(ec);
+    });
 }
 
 void
@@ -101,11 +340,13 @@ plain_stream_impl::set_options()
 void
 plain_stream_impl::async_connect(
   const asio::ip::tcp::resolver::results_type::endpoint_type& endpoint,
+  const std::string& /* hostname */,
   utils::movable_function<void(std::error_code)>&& handler)
 {
   if (!stream_) {
     id_ = uuid::to_string(uuid::random());
     stream_ = std::make_shared<asio::ip::tcp::socket>(strand_);
+    inherited_.add(stream_);
   }
   return stream_->async_connect(endpoint,
                                 [stream = stream_, handler = std::move(handler)](auto ec) {
@@ -147,6 +388,7 @@ tls_stream_impl::tls_stream_impl(asio::io_context& ctx, tls_context_provider& tl
       std::make_shared<asio::ssl::stream<asio::ip::tcp::socket>>(asio::ip::tcp::socket(strand_),
                                                                  *tls_.get_ctx()))
 {
+  inherited_.add(stream_);
 }
 
 auto
@@ -178,12 +420,19 @@ tls_stream_impl::close(utils::movable_function<void(std::error_code)>&& handler)
   if (!stream_) {
     return handler(asio::error::bad_descriptor);
   }
-  return asio::post(strand_, [stream = std::move(stream_), handler = std::move(handler)]() {
-    asio::error_code ec{};
-    stream->lowest_layer().shutdown(asio::socket_base::shutdown_both, ec);
-    stream->lowest_layer().close(ec);
-    handler(ec);
-  });
+  return asio::post(
+    strand_, [stream = std::move(stream_), owner_pid = owner_pid_, handler = std::move(handler)]() {
+      asio::error_code ec{};
+      // See plain_stream_impl::close(): the inheritance test belongs here, not at
+      // post time.
+      if (owner_pid != utils::current_process_id()) {
+        detach_inherited_socket(stream->lowest_layer(), ec);
+      } else {
+        stream->lowest_layer().shutdown(asio::socket_base::shutdown_both, ec);
+        stream->lowest_layer().close(ec);
+      }
+      handler(ec);
+    });
 }
 
 void
@@ -199,21 +448,33 @@ tls_stream_impl::set_options()
 
 void
 tls_stream_impl::async_connect(const asio::ip::tcp::resolver::results_type::endpoint_type& endpoint,
+                               const std::string& hostname,
                                utils::movable_function<void(std::error_code)>&& handler)
 {
   if (!stream_) {
     id_ = uuid::to_string(uuid::random());
     stream_ = std::make_shared<asio::ssl::stream<asio::ip::tcp::socket>>(
       asio::ip::tcp::socket(strand_), *tls_.get_ctx());
+    inherited_.add(stream_);
   }
   return stream_->lowest_layer().async_connect(
-    endpoint, [stream = stream_, handler = std::move(handler)](std::error_code ec_connect) mutable {
+    endpoint,
+    // hostname is copied into the closure (owning std::string) so the async
+    // handler does not depend on the caller's argument lifetime.
+    [stream = stream_, hostname = hostname, handler = std::move(handler)](
+      std::error_code ec_connect) mutable {
       if (ec_connect == asio::error::operation_aborted) {
         return;
       }
       if (ec_connect) {
         return handler(ec_connect);
       }
+      // Verify the server's identity against `hostname` (SAN/CN) and set SNI
+      // before the handshake. Without this, verify_peer only checks that the
+      // certificate chains to a trusted CA, not that it identifies this host.
+      if (auto ec_identity = configure_tls_handshake(*stream, hostname); ec_identity) {
+        return handler(ec_identity);
+      }
       stream->async_handshake(
         asio::ssl::stream_base::client,
         [stream, handler = std::move(handler)](std::error_code ec_handshake) mutable {

data/ext/couchbase/core/io/streams.hxx

@@ -24,10 +24,12 @@
 #include <asio/error.hpp>
 #include <asio/io_context.hpp>
 #include <asio/ip/tcp.hpp>
+#include <asio/ip/udp.hpp>
 #include <asio/ssl/context.hpp>
 #include <asio/ssl/stream.hpp>
 #include <asio/strand.hpp>
 
+#include <cstdint>
 #include <string>
 #include <vector>
 
@@ -59,12 +61,58 @@ async_resolve(ip_protocol protocol,
   return resolver.async_resolve(hostname, service, std::forward<Handler>(handler));
 }
 
+/**
+ * Configure server-identity verification on a client TLS stream prior to the
+ * handshake: install a hostname (SAN/CN) verification callback and set the SNI
+ * server name. When the underlying context is in verify_none mode the callback
+ * has no effect, but SNI is still set.
+ *
+ * Without this, asio::ssl::verify_peer only validates that the certificate
+ * chains to a trusted CA -- it does NOT check that the certificate identifies
+ * the host being contacted (CWE-297).
+ *
+ * @param stream the client TLS stream that has connected but not yet handshaken
+ * @param hostname the canonical hostname the caller intended to reach; must not
+ * be empty (an empty hostname is rejected to avoid silently falling back to
+ * CA-only validation)
+ * @return a non-empty error_code if the hostname is empty, SNI could not be set,
+ * or the hostname-verification callback could not be installed; empty on success
+ */
+[[nodiscard]] auto
+configure_tls_handshake(asio::ssl::stream<asio::ip::tcp::socket>& stream,
+                        const std::string& hostname) -> std::error_code;
+
+class inherited_stream_registry;
+
+// For a forked child, after io_context::notify_fork(fork_child) and before anything runs on ctx:
+// releases and closes every registered stream and socket on ctx that another process opened,
+// without shutdown(2). No pending operation on them can then read or write a connection the
+// parent is still using. The DNS client registers its sockets through register_inherited_socket().
+void
+detach_inherited_sockets(asio::io_context& ctx);
+
+// Records a socket on ctx that is not a stream, so that detach_inherited_sockets() in a forked
+// child releases it. Held weakly: the registry does not keep the socket alive.
+void
+register_inherited_socket(asio::io_context& ctx,
+                          const std::shared_ptr<asio::ip::tcp::socket>& socket);
+void
+register_inherited_socket(asio::io_context& ctx,
+                          const std::shared_ptr<asio::ip::udp::socket>& socket);
+
 class stream_impl
 {
 protected:
+  inherited_stream_registry& inherited_;
   asio::strand<asio::io_context::executor_type> strand_;
   bool tls_;
   std::string id_{};
+  // Identifies the process that opened this socket. close() compares it against
+  // the process running at that moment to tell a socket inherited across fork(2)
+  // apart from one this process opened itself, because an inherited socket must not
+  // be shut down: shutdown(2) acts on the open file description, which fork(2)
+  // shares, so it would tear down a connection the parent is still using.
+  std::int64_t owner_pid_;
 
 public:
   stream_impl(asio::io_context& ctx, bool is_tls);
@@ -88,7 +136,10 @@ public:
 
   virtual void set_options() = 0;
 
+  // `hostname` is the canonical name the caller intends to reach; TLS streams use
+  // it to verify the server certificate's identity (SAN/CN) and to set SNI.
   virtual void async_connect(const asio::ip::tcp::resolver::results_type::endpoint_type& endpoint,
+                             const std::string& hostname,
                              utils::movable_function<void(std::error_code)>&& handler) = 0;
 
   virtual void async_write(
@@ -117,6 +168,7 @@ public:
   void set_options() override;
 
   void async_connect(const asio::ip::tcp::resolver::results_type::endpoint_type& endpoint,
+                     const std::string& hostname,
                      utils::movable_function<void(std::error_code)>&& handler) override;
 
   void async_write(std::vector<asio::const_buffer>& buffers,
@@ -145,6 +197,7 @@ public:
   void set_options() override;
 
   void async_connect(const asio::ip::tcp::resolver::results_type::endpoint_type& endpoint,
+                     const std::string& hostname,
                      utils::movable_function<void(std::error_code)>&& handler) override;
 
   void async_write(std::vector<asio::const_buffer>& buffers,

data/ext/couchbase/core/logger/logger.cxx

@@ -62,6 +62,10 @@ std::atomic_int file_logger_version{ 0 };
 std::shared_ptr<couchbase::core::logger::log_callback> log_callback{};
 std::mutex log_callback_mutex;
 std::atomic_int log_callback_version{ 0 };
+// Mirrors "is log_callback set?" so the hot-path predicate can be answered with a single atomic
+// load, without copying the owning shared_ptr. Updated under log_callback_mutex alongside
+// log_callback.
+std::atomic<bool> log_callback_present{ false };
 
 auto
 get_file_logger() -> std::shared_ptr<spdlog::logger>
@@ -94,6 +98,7 @@ update_callback_logger(const std::shared_ptr<couchbase::core::logger::log_callba
 {
   const std::scoped_lock lock(log_callback_mutex);
   log_callback = new_callback;
+  log_callback_present.store(new_callback != nullptr, std::memory_order_relaxed);
   ++log_callback_version;
 }
 
@@ -196,6 +201,14 @@ should_log(level lvl) -> bool
   return false;
 }
 
+auto
+has_custom_callback() -> bool
+{
+  // A log macro asks this on every call; answer it with a single atomic load rather than copying
+  // the owning shared_ptr that get_custom_callback() returns.
+  return log_callback_present.load(std::memory_order_relaxed);
+}
+
 namespace detail
 {
 void
@@ -252,6 +265,29 @@ is_initialized() -> bool
   return get_file_logger() != nullptr;
 }
 
+namespace
+{
+std::atomic<bool> log_redaction_enabled{ false };
+} // namespace
+
+void
+set_log_redaction(bool enable)
+{
+  const auto was_enabled = log_redaction_enabled.exchange(enable, std::memory_order_relaxed);
+  if (enable && !was_enabled) {
+    CB_LOG_INFO_RAW(
+      "Log redaction enabled: sensitive values are wrapped in redaction tags. This log still "
+      "contains identifying information until those tags are processed. Once redacted, diagnosis "
+      "and support of issues may be challenging or not possible");
+  }
+}
+
+auto
+is_log_redaction_enabled() -> bool
+{
+  return log_redaction_enabled.load(std::memory_order_relaxed);
+}
+
 auto
 create_file_logger_impl(const std::string& logger_name, const configuration& logger_settings)
   -> std::pair<std::optional<std::string>, std::shared_ptr<spdlog::logger>>

data/ext/couchbase/core/logger/logger.hxx

@@ -181,6 +181,14 @@ get_lowest_log_level() -> level;
 auto
 should_log(level lvl) -> bool;
 
+/**
+ * @return true if a custom log callback is currently registered. Used to decide whether log
+ * arguments need to be formatted at all, independently of the severity level (the callback does
+ * its own level filtering).
+ */
+auto
+has_custom_callback() -> bool;
+
 auto
 should_log_protocol() -> bool;
 
@@ -265,6 +273,29 @@ shutdown();
  */
 auto
 is_initialized() -> bool;
+
+/**
+ * Enable or disable log redaction.
+ *
+ * Redaction state is process-wide. When enabled, values wrapped by the helpers in
+ * `core/logger/redaction.hxx` are surrounded by `<ud>`/`<md>`/`<sd>` tags so that an external tool
+ * can strip or hash them; the SDK never removes or obscures anything itself.
+ *
+ * Connection-context log prefixes bake their tags in at construction, so redaction must be enabled
+ * *before* connecting for those prefixes to be tagged. Enabling it later affects only prefixes
+ * built afterwards.
+ *
+ * Enabling emits a one-time breadcrumb recording that the log carries redaction tags and still
+ * needs processing. That breadcrumb is dropped if no logger has been initialized yet.
+ */
+void
+set_log_redaction(bool enable);
+
+/**
+ * @return whether log redaction is currently enabled
+ */
+auto
+is_log_redaction_enabled() -> bool;
 } // namespace couchbase::core::logger
 
 #if defined(__GNUC__) || defined(__clang__)
@@ -279,10 +310,17 @@ is_initialized() -> bool;
  */
 #define COUCHBASE_LOG(file, line, function, severity, ...)                                         \
   do {                                                                                             \
-    couchbase::core::logger::log_custom_logger(file, line, function, severity, __VA_ARGS__);       \
-                                                                                                   \
-    if (couchbase::core::logger::should_log(severity)) {                                           \
-      couchbase::core::logger::log(file, line, function, severity, __VA_ARGS__);                   \
+    const bool cb_log_to_file_ = couchbase::core::logger::should_log(severity);                    \
+    const bool cb_log_to_custom_ = couchbase::core::logger::has_custom_callback();                 \
+    if (cb_log_to_file_ || cb_log_to_custom_) {                                                    \
+      auto cb_log_message_ = ::fmt::format(__VA_ARGS__);                                           \
+      if (cb_log_to_custom_) {                                                                     \
+        couchbase::core::logger::detail::log_custom_logger(                                        \
+          file, line, function, severity, cb_log_message_);                                        \
+      }                                                                                            \
+      if (cb_log_to_file_) {                                                                       \
+        couchbase::core::logger::detail::log(file, line, function, severity, cb_log_message_);     \
+      }                                                                                            \
     }                                                                                              \
   } while (false)
 
@@ -343,8 +381,10 @@ is_initialized() -> bool;
  */
 #define COUCHBASE_LOG_RAW(file, line, function, severity, msg)                                     \
   do {                                                                                             \
-    couchbase::core::logger::log_custom_logger(file, line, function, severity, msg);               \
-                                                                                                   \
+    /* msg is already a string; deliver it directly (no fmt::format), gated like COUCHBASE_LOG. */ \
+    if (couchbase::core::logger::has_custom_callback()) {                                          \
+      couchbase::core::logger::detail::log_custom_logger(file, line, function, severity, msg);     \
+    }                                                                                              \
     if (couchbase::core::logger::should_log(severity)) {                                           \
       couchbase::core::logger::detail::log(file, line, function, severity, msg);                   \
     }                                                                                              \