couchbase 3.8.1 → 3.9.0

data/ext/couchbase/core/bucket.cxx

@@ -17,6 +17,8 @@
 
 #include "bucket.hxx"
 
+#include "bucket_unit_test_api.hxx"
+
 #include "collection_id_cache_entry.hxx"
 #include "core/app_telemetry_meter.hxx"
 #include "core/cluster_options.hxx"
@@ -26,6 +28,7 @@
 #include "core/error_context/key_value_status_code.hxx"
 #include "core/io/mcbp_message.hxx"
 #include "core/logger/logger.hxx"
+#include "core/logger/redaction.hxx"
 #include "core/mcbp/codec.hxx"
 #include "core/metrics/meter_wrapper.hxx"
 #include "core/protocol/client_opcode.hxx"
@@ -33,6 +36,7 @@
 #include "core/protocol/hello_feature.hxx"
 #include "core/response_handler.hxx"
 #include "core/service_type.hxx"
+#include "core/tracing/constants.hxx"
 #include "core/tracing/tracer_wrapper.hxx"
 #include "core/utils/movable_function.hxx"
 #include "dispatcher.hxx"
@@ -45,6 +49,7 @@
 #include "protocol/cmd_get_cluster_config.hxx"
 #include "retry_orchestrator.hxx"
 
+#include <couchbase/build_config.hxx>
 #include <couchbase/error_codes.hxx>
 #include <couchbase/retry_reason.hxx>
 #include <couchbase/retry_strategy.hxx>
@@ -94,7 +99,7 @@ public:
               tls_context_provider& tls)
     : client_id_{ std::move(client_id) }
     , name_{ std::move(name) }
-    , log_prefix_{ fmt::format("[{}/{}]", client_id_, name_) }
+    , log_prefix_{ fmt::format("[{}/{}]", client_id_, logger::metadata(name_)) }
     , tracer_{ std::move(tracer) }
     , meter_{ std::move(meter) }
     , orphan_reporter_{ std::move(orphan_reporter) }
@@ -122,7 +127,11 @@ public:
     // TODO(SA): copy from mcbp_command, subject to refactor later
 #ifdef COUCHBASE_CXX_CLIENT_CREATE_OPERATION_SPAN_IN_CORE
     metrics::metric_attributes attrs{
-      service_type::key_value, fmt::format("{}", req->command_), ec, name_, req->scope_name_,
+      tracing::service::key_value,
+      fmt::format("{}", req->command_),
+      metrics::standardized_error_type(ec),
+      name_,
+      req->scope_name_,
       req->collection_name_,
     };
     meter_->record_value(std::move(attrs), req->dispatched_time_);
@@ -141,8 +150,13 @@ public:
         // self->span_->add_tag(tracing::attributes::orphan, "canceled");
         return req->try_callback(resp, ec);
       }
-      backoff_and_retry(
-        req, reason == retry_reason::do_not_retry ? retry_reason::node_not_available : reason);
+      if (!backoff_and_retry(req,
+                             reason == retry_reason::do_not_retry ? retry_reason::node_not_available
+                                                                  : reason)) {
+        // The retry was refused, so nothing else will carry this request: complete it here, as
+        // every other caller of backoff_and_retry() does.
+        req->try_callback(resp, ec);
+      }
       return;
     }
     key_value_status_code status{ key_value_status_code::unknown };
@@ -157,6 +171,58 @@ public:
       reason = retry_reason::key_value_error_map_retry_indicated;
     } else {
       switch (status) {
+        case key_value_status_code::unknown_collection:
+          /*
+           * SDK-RFC 0049 (SDK3 Retry Handling), section "Response Code Retry List":
+           * > Explicitly excluded here is the GetCollectionID KV command, since it returns 0x88 if
+           * > the collection is not found.
+           *
+           * The request completes with collection_not_found.
+           */
+          if (req->command_ == protocol::client_opcode::get_collection_id) {
+            reason = retry_reason::do_not_retry;
+            break;
+          }
+          /**
+           * The server does not recognise the collection id we sent. This covers two cases:
+           *
+           *   1. The collection was dropped and recreated, so its id changed. The cached id we sent
+           *      is now stale and re-sending it would loop forever. We must invalidate the cache
+           *      and re-resolve the id via GetCollectionID before retrying.
+           *   2. The node's collection manifest simply lags ns_server's view: the id is still valid
+           *      but has not propagated to this particular KV node yet. Here re-sending the same id
+           *      succeeds once the manifest catches up, bounded by the operation timeout.
+           *
+           * Prefer re-resolution: if the request was dispatched through the collection-id cache the
+           * hook invalidates the cached id and schedules a re-resolution retry (covering case 1,
+           * and case 2 too since GetCollectionID re-fetches). This matches the legacy mcbp_command
+           * path, which handled both via handle_unknown_collection(); when KV ops were migrated to
+           * this path that handling was dropped and the status surfaced directly as
+           * collection_not_found, which consumers such as the range scan orchestrator treat as
+           * fatal.
+           *
+           * If there is no hook (request bypassed the cid cache) or re-resolution gave up, fall
+           * back to a bounded collection-outdated retry with the same id, which still recovers
+           * case 2.
+           */
+          if (req->on_unknown_collection_ && req->on_unknown_collection_()) {
+            return;
+          }
+          reason = retry_reason::key_value_collection_outdated;
+          break;
+        case key_value_status_code::config_only:
+          /**
+           * The node has a configuration but is not serving data operations yet (e.g. during
+           * warmup or rebalance). Mirror the legacy path: fetch a fresh configuration and retry so
+           * the request is routed to a node that can serve it.
+           */
+          CB_LOG_DEBUG(
+            "{} server returned config_only status, meaning that the node does not serve "
+            "data operations, requesting new configuration and retrying",
+            log_prefix_);
+          fetch_config();
+          reason = retry_reason::service_response_code_indicated;
+          break;
         case key_value_status_code::locked:
           if (req->command_ != protocol::client_opcode::unlock) {
             /**
@@ -185,6 +251,7 @@ public:
   }
 
   void handle_response(std::shared_ptr<mcbp::queue_request> req,
+                       std::optional<io::mcbp_session> /* session */,
                        std::error_code error,
                        retry_reason reason,
                        io::mcbp_message msg,
@@ -213,21 +280,38 @@ public:
         if (ec == errc::common::request_canceled) {
           return req->cancel(ec);
         }
-        self->direct_dispatch(req);
+        // This runs after direct_dispatch() returned, so its error code has nobody left to report
+        // to. Complete the request here rather than leave it to its deadline.
+        if (auto rc = self->direct_dispatch(req); rc) {
+          req->try_callback({}, rc);
+        }
       });
     }
 
     req->dispatched_time_ = std::chrono::steady_clock::now();
 
-    auto session = route_request(req);
-    if (!session || !session->has_config()) {
-      return defer_command([self = shared_from_this(), req](std::error_code ec) mutable {
+    auto routed = route_request(req);
+    if (!routed.session || !routed.session->has_config()) {
+      if (!routed.connect_pending) {
+        // No bootstrap was started for this node, and only a bootstrap
+        // completion or a bucket close drains the deferred queue. Deferring
+        // here would leave the request waiting for its deadline, so retry
+        // instead: that re-routes against the current configuration.
+        if (backoff_and_retry(req, retry_reason::node_not_available)) {
+          return {};
+        }
+        return errc::common::service_not_available;
+      }
+      return defer_command([self = shared_from_this(), req](std::error_code ec) {
         if (ec == errc::common::request_canceled) {
           return req->cancel(ec);
         }
-        self->direct_dispatch(std::move(req));
+        if (auto rc = self->direct_dispatch(req); rc) {
+          req->try_callback({}, rc);
+        }
       });
     }
+    auto session = routed.session;
     if (session->is_stopped()) {
       if (backoff_and_retry(req, retry_reason::node_not_available)) {
         return {};
@@ -244,7 +328,7 @@ public:
   {
     auto handle_error = [is_retry, req](std::error_code ec) {
       // We only want to log an error on retries if the error isn't cancelled.
-      if (!is_retry || (is_retry && ec != errc::common::request_canceled)) {
+      if (!is_retry || ec != errc::common::request_canceled) {
         CB_LOG_ERROR("reschedule failed, failing request ({})", ec.message());
       }
 
@@ -253,15 +337,29 @@ public:
 
     CB_LOG_DEBUG("request being re-queued. opaque={}, opcode={}", req->opaque_, req->command_);
 
-    auto session = route_request(req);
-    if (!session || !session->has_config()) {
-      return defer_command([self = shared_from_this(), req](std::error_code ec) {
+    auto routed = route_request(req);
+    if (!routed.session || !routed.session->has_config()) {
+      if (!routed.connect_pending) {
+        // As in direct_dispatch(): no bootstrap is pending for this node, so a
+        // deferred request would wait for its deadline instead of being sent.
+        if (backoff_and_retry(req, retry_reason::node_not_available)) {
+          return {};
+        }
+        // A re-queue is driven by a backoff timer that ignores what this
+        // returns, so the request has to be completed here.
+        handle_error(errc::common::service_not_available);
+        return errc::common::service_not_available;
+      }
+      return defer_command([self = shared_from_this(), req, handle_error](std::error_code ec) {
         if (ec == errc::common::request_canceled) {
           return req->cancel(ec);
         }
-        self->direct_dispatch(req);
+        if (auto rc = self->direct_dispatch(req); rc) {
+          handle_error(rc);
+        }
       });
     }
+    auto session = routed.session;
     if (session->is_stopped()) {
       if (backoff_and_retry(req, retry_reason::node_not_available)) {
         return {};
@@ -302,23 +400,55 @@ public:
     -> bool
   {
     auto action = retry_orchestrator::should_retry(request, reason);
-    auto retried = action.need_to_retry();
-    if (retried) {
-      auto timer = std::make_shared<asio::steady_timer>(ctx_);
-      timer->expires_after(action.duration());
-      timer->async_wait([self = shared_from_this(), request](auto error) {
-        if (error == asio::error::operation_aborted) {
-          return;
-        }
-        self->direct_re_queue(request, true);
-      });
-      request->set_retry_backoff(timer);
+    if (!action.need_to_retry()) {
+      return false;
+    }
+
+    // Park the request where close() can reach it. The timer below is the only
+    // thing that would re-dispatch it, and asio abandons a pending wait when the
+    // io_context goes away rather than completing it, so a request reachable only
+    // from that handler is neither completed nor released. A queue also lets
+    // cancel() take it back out, since a request de-queues from whichever queue
+    // holds it.
+    if (auto ec = retry_backoff_queue_.push(request, 0); ec) {
+      // Nothing will retry it: the bucket has closed, or it is already cancelled.
+      request->try_callback({}, errc::common::request_canceled);
+      return true;
     }
-    return retried;
+
+    auto timer = std::make_shared<asio::steady_timer>(ctx_);
+    timer->expires_after(action.duration());
+    timer->async_wait([self = shared_from_this(), request](auto error) {
+      if (!self->retry_backoff_queue_.remove(request)) {
+        // close() completed it, or it was cancelled: either way it is done.
+        return;
+      }
+      if (error == asio::error::operation_aborted) {
+        request->try_callback({}, errc::common::request_canceled);
+        return;
+      }
+      self->direct_re_queue(request, true);
+    });
+    request->set_retry_backoff(timer);
+    return true;
   }
 
-  auto route_request(const std::shared_ptr<mcbp::queue_request>& req)
-    -> std::optional<io::mcbp_session>
+  struct routed_session {
+    std::optional<io::mcbp_session> session{};
+
+    /**
+     * False when no session exists for the node and no bootstrap was started
+     * for it, so nothing will drain the deferred queue. The caller must retry
+     * rather than defer; see connect_session().
+     *
+     * The default is false. route_request() returns a default-constructed
+     * value when the configuration names no server for the vbucket, and a
+     * request deferred on that has nothing to wait for.
+     */
+    bool connect_pending{ false };
+  };
+
+  auto route_request(const std::shared_ptr<mcbp::queue_request>& req) -> routed_session
   {
     if (req->key_.empty()) {
       if (auto server = server_by_vbucket(req->vbucket_, req->replica_index_); server) {
@@ -328,7 +458,7 @@ public:
       req->vbucket_ = partition;
       return find_or_connect_session_by_index(server.value());
     }
-    return std::nullopt;
+    return {};
   }
 
   [[nodiscard]] auto server_by_vbucket(std::uint16_t vbucket, std::size_t node_index)
@@ -351,6 +481,12 @@ public:
     return { 0, std::nullopt };
   }
 
+  [[nodiscard]] auto config_snapshot() const -> std::shared_ptr<topology::configuration>
+  {
+    const std::scoped_lock lock(config_mutex_);
+    return config_;
+  }
+
   auto config_rev() const -> std::string
   {
     const std::scoped_lock lock(config_mutex_);
@@ -370,11 +506,33 @@ public:
     return { 0, std::nullopt };
   }
 
-  void connect_session(std::size_t index)
+  /**
+   * A command deferred for this index is only ever dispatched or completed by
+   * one of two events: a bootstrap finishing, or the bucket closing.
+   *
+   * @return true when one of them is coming. Either a bootstrap is now pending
+   * for this index, or the bucket is closed and defer_command() completes the
+   * command with a cancellation. False when neither is, in which case the
+   * caller must retry: a command deferred now would wait for its deadline.
+   */
+  [[nodiscard]] auto connect_session(std::size_t index) -> bool
   {
     const std::scoped_lock lock(config_mutex_, sessions_mutex_);
     if (!config_) {
-      return;
+      return false;
+    }
+    if (closed_) {
+      // Do not connect a session into a bucket that is being torn down: close() has already stopped
+      // sessions_ and will not run again, so a session added here would never be stopped and would
+      // strand the whole bucket/session graph as a pure cycle (see the closed_ guard in bootstrap()
+      // and update_config()).
+      return true;
+    }
+
+    if (index >= config_->nodes.size()) {
+      // The index was resolved against an earlier configuration and the node it
+      // named is no longer in the topology, so there is nothing to connect to.
+      return false;
     }
 
     const auto& node = config_->nodes[index];
@@ -383,7 +541,9 @@ public:
     auto port = node.port_or(
       origin_.options().network, service_type::key_value, origin_.options().enable_tls, 0);
     if (port == 0) {
-      return;
+      // The node is in the topology but advertises no key-value port, so there
+      // is no session to open for it either.
+      return false;
     }
 
     const couchbase::core::origin origin(origin_.credentials(), hostname, port, origin_.options());
@@ -393,13 +553,12 @@ public:
             client_id_, node.node_uuid, ctx_, tls_, origin, state_listener_, name_, known_features_)
         : io::mcbp_session(
             client_id_, node.node_uuid, ctx_, origin, state_listener_, name_, known_features_);
-    CB_LOG_DEBUG(R"({} rev={}, connect idx={}, session="{}", address="{}:{}")",
+    CB_LOG_DEBUG(R"({} rev={}, connect idx={}, session="{}", address="{}")",
                  log_prefix_,
                  config_->rev_str(),
                  node.index,
                  session.id(),
-                 hostname,
-                 port);
+                 logger::system_data(fmt::format("{}:{}", hostname, port)));
     session.bootstrap(
       [self = shared_from_this(), session](std::error_code err,
                                            topology::configuration cfg) mutable -> void {
@@ -415,6 +574,7 @@ public:
       },
       true);
     sessions_.insert_or_assign(index, std::move(session));
+    return true;
   }
 
   void restart_sessions()
@@ -423,6 +583,13 @@ public:
     if (!config_) {
       return;
     }
+    if (closed_) {
+      // remove_session() posts restart_sessions() whenever a session drops. If that post is
+      // serviced after close() has torn the bucket down, recreating sessions here would strand them
+      // (nothing stops a session added after close()), leaking the whole bucket/session graph as a
+      // pure cycle. close() sets closed_ before it swaps and stops sessions_, so bail out.
+      return;
+    }
 
     std::size_t kv_node_index{ 0 };
     for (std::size_t index = 0; index < config_->nodes.size(); ++index) {
@@ -445,12 +612,11 @@ public:
         if (auto found_kv_node_index = ptr->first; found_kv_node_index != kv_node_index) {
           if (auto current = sessions_.find(kv_node_index); current == sessions_.end()) {
             CB_LOG_WARNING(
-              R"({} KV node index mismatch: config rev={} states that address="{}:{}" should be at idx={}, )"
+              R"({} KV node index mismatch: config rev={} states that address="{}" should be at idx={}, )"
               R"(but it is at idx={} ("{}"). Moving session to idx={}.)",
               log_prefix_,
               config_->rev_str(),
-              hostname,
-              port,
+              logger::system_data(fmt::format("{}:{}", hostname, port)),
               kv_node_index,
               found_kv_node_index,
               ptr->second.id(),
@@ -459,17 +625,16 @@ public:
             sessions_.erase(ptr);
           } else {
             CB_LOG_WARNING(
-              R"({} KV node index mismatch: config rev={} states that address="{}:{}" should be at idx={}, )"
+              R"({} KV node index mismatch: config rev={} states that address="{}" should be at idx={}, )"
               R"(but it is at idx={} ("{}"). Slot with idx={} is holds session with address="{}" ("{}"), swapping them.)",
               log_prefix_,
               config_->rev_str(),
-              hostname,
-              port,
+              logger::system_data(fmt::format("{}:{}", hostname, port)),
               kv_node_index,
               found_kv_node_index,
               ptr->second.id(),
               kv_node_index,
-              current->second.bootstrap_address(),
+              logger::system_data(current->second.bootstrap_address()),
               current->second.id());
             std::swap(current->second, ptr->second);
           }
@@ -491,13 +656,12 @@ public:
                              known_features_)
           : io::mcbp_session(
               client_id_, node.node_uuid, ctx_, origin, state_listener_, name_, known_features_);
-      CB_LOG_DEBUG(R"({} rev={}, restart idx={}, session="{}", address="{}:{}")",
+      CB_LOG_DEBUG(R"({} rev={}, restart idx={}, session="{}", address="{}")",
                    log_prefix_,
                    config_->rev_str(),
                    node.index,
                    session.id(),
-                   hostname,
-                   port);
+                   logger::system_data(fmt::format("{}:{}", hostname, port)));
       session.bootstrap(
         [self = shared_from_this(), session](std::error_code err,
                                              topology::configuration cfg) mutable {
@@ -523,12 +687,11 @@ public:
     const std::scoped_lock lock(sessions_mutex_);
     for (auto ptr = sessions_.cbegin(); ptr != sessions_.cend();) {
       if (ptr->second.id() == id) {
-        CB_LOG_DEBUG(R"({} removed session id="{}", address="{}", bootstrap_address="{}:{}")",
+        CB_LOG_DEBUG(R"({} removed session id="{}", address="{}", bootstrap_address="{}")",
                      log_prefix_,
                      ptr->second.id(),
-                     ptr->second.remote_address(),
-                     ptr->second.bootstrap_hostname(),
-                     ptr->second.bootstrap_port());
+                     logger::system_data(ptr->second.remote_address()),
+                     logger::system_data(ptr->second.bootstrap_address()));
         ptr = sessions_.erase(ptr);
         found = true;
       } else {
@@ -545,6 +708,17 @@ public:
 
   void bootstrap(utils::movable_function<void(std::error_code, topology::configuration)>&& handler)
   {
+    if (closed_) {
+      // The bucket was closed before this bootstrap even started -- e.g. the cluster was torn down
+      // in the window between cluster_impl::open_bucket() inserting the bucket into buckets_ and
+      // calling bootstrap() (the buckets_ lock is released in between). close() is one-shot and
+      // will not run again, so registering a config listener or starting a session here would never
+      // be undone: the session, its self-capturing bootstrap continuation, and this bucket would be
+      // stranded as a pure cycle that LeakSanitizer/Valgrind report as leaked, and the continuation
+      // (which a scan/ping/etc. may be parked behind) would never fire. Bail before touching any
+      // shared state.
+      return handler(errc::common::request_canceled, {});
+    }
     if (state_listener_) {
       state_listener_->register_config_listener(shared_from_this());
     }
@@ -553,28 +727,68 @@ public:
         ? io::mcbp_session(
             client_id_, {}, ctx_, tls_, origin_, state_listener_, name_, known_features_)
         : io::mcbp_session(client_id_, {}, ctx_, origin_, state_listener_, name_, known_features_);
+    {
+      // Publish the in-flight session so close() can stop it if the cluster is torn down before
+      // this bootstrap completes; otherwise the continuation below (which captures new_session)
+      // would keep it alive forever. See bootstrapping_session_.
+      const std::scoped_lock lock(sessions_mutex_);
+      if (closed_) {
+        // close() ran concurrently after the entry check above (it sets closed_ before it swaps and
+        // stops sessions_ under sessions_mutex_). Do not publish a session close() has already
+        // swept past -- nothing would ever stop it. The bootstrap continuation is not registered
+        // yet, so returning here strands nothing.
+        return handler(errc::common::request_canceled, {});
+      }
+      bootstrapping_session_ = new_session;
+    }
     new_session.bootstrap([self = shared_from_this(), new_session, h = std::move(handler)](
                             std::error_code ec, topology::configuration cfg) mutable {
       if (ec) {
         CB_LOG_WARNING(R"({} failed to bootstrap session ec={}, bucket="{}")",
                        new_session.log_prefix(),
                        ec.message(),
-                       self->name_);
+                       logger::metadata(self->name_));
         self->remove_session(new_session.id());
-      } else {
+        {
+          const std::scoped_lock lock(self->sessions_mutex_);
+          self->bootstrapping_session_.reset();
+        }
+        // Deliver the failure inline rather than through the asio::post below: on cluster teardown
+        // the io_context is stopped and any queued post is discarded, so a posted completion would
+        // be dropped -- a std::future caller then observes broken_promise and a callback caller
+        // hangs. On the error path the handler only reports the failure to the operation (it does
+        // not re-enter the bucket under a held lock here), so completing it synchronously is safe.
+        return h(ec, cfg);
+      }
+      {
         const std::size_t this_index = new_session.index();
-        new_session.on_configuration_update(self);
-        new_session.on_stop([id = new_session.id(), self]() {
-          self->remove_session(id);
-        });
-
+        bool established{ false };
         {
+          // Establish the session and stop tracking it as bootstrapping in one critical section, so
+          // a concurrent close() finds it in exactly one of the two containers (never neither).
           const std::scoped_lock lock(self->sessions_mutex_);
-          self->sessions_.insert_or_assign(this_index, std::move(new_session));
+          self->bootstrapping_session_.reset();
+          if (self->closed_) {
+            // The bucket was closed while this session was still bootstrapping (close() sets
+            // closed_ before it swaps the session maps). Do not register callbacks or add it to
+            // sessions_: a session established after close() would be stranded, and its on_stop
+            // handler could post restart_sessions() during shutdown. Stop it here instead -- no
+            // on_stop is registered, so stopping under the lock cannot re-enter remove_session().
+            new_session.stop(retry_reason::do_not_retry);
+          } else {
+            new_session.on_configuration_update(self);
+            new_session.on_stop([id = new_session.id(), self]() {
+              self->remove_session(id);
+            });
+            self->sessions_.insert_or_assign(this_index, std::move(new_session));
+            established = true;
+          }
+        }
+        if (established) {
+          self->update_config(cfg);
+          self->drain_deferred_queue({});
+          self->poll_config({});
         }
-        self->update_config(cfg);
-        self->drain_deferred_queue({});
-        self->poll_config({});
       }
       asio::post(
         asio::bind_executor(self->ctx_, [h = std::move(h), ec, cfg = std::move(cfg)]() mutable {
@@ -601,8 +815,12 @@ public:
       }
       return handler(errc::network::configuration_not_available, nullptr);
     }
-    const std::scoped_lock lock(deferred_commands_mutex_);
-    deferred_commands_.emplace(
+    // Park the continuation until the bucket is configured. defer_command re-checks closed_ under
+    // deferred_commands_mutex_, so a continuation that races close() is either drained or completed
+    // with request_canceled instead of being stranded. Stranding would leak the whole cluster
+    // graph: the continuation strongly captures the cluster_impl (core), which owns this bucket,
+    // which owns the deferred queue that would hold the orphaned continuation (CXXCBC-846).
+    defer_command(
       [self = shared_from_this(), handler = std::move(handler)](std::error_code ec) mutable {
         if (ec == errc::common::request_canceled || self->closed_ || !self->configured_) {
           return handler(errc::network::configuration_not_available, nullptr);
@@ -704,6 +922,14 @@ public:
 
     drain_deferred_queue(errc::common::request_canceled);
 
+    // A request waiting for a retry backoff is only ever re-dispatched by its
+    // timer, which a closed bucket never fires. Complete them here, for the same
+    // reason the deferred queue is drained above.
+    retry_backoff_queue_.close();
+    retry_backoff_queue_.drain([](const auto& request) {
+      request->try_callback({}, errc::common::request_canceled);
+    });
+
     if (state_listener_ != nullptr) {
       state_listener_->unregister_config_listener(shared_from_this());
     }
@@ -714,13 +940,20 @@ public:
     }
 
     std::map<size_t, io::mcbp_session> old_sessions;
+    std::optional<io::mcbp_session> bootstrapping_session;
     {
       const std::scoped_lock lock(sessions_mutex_);
       std::swap(old_sessions, sessions_);
+      std::swap(bootstrapping_session, bootstrapping_session_);
     }
     for (auto& [index, session] : old_sessions) {
       session.stop(retry_reason::do_not_retry);
     }
+    // Stop the session that is still bootstrapping (if any): this completes its bootstrap with an
+    // error, releasing the continuation that would otherwise strand the session and this bucket.
+    if (bootstrapping_session) {
+      bootstrapping_session->stop(retry_reason::do_not_retry);
+    }
   }
 
   /**
@@ -831,8 +1064,19 @@ public:
         }
       }
     }
+    std::vector<io::mcbp_session> dropped_sessions{};
     if (!added.empty() || !removed.empty() || sequence_changed) {
       const std::scoped_lock lock(sessions_mutex_);
+      if (closed_) {
+        // The bucket was closed concurrently (close() sets closed_ before it swaps and stops
+        // sessions_ under sessions_mutex_). Do not rebuild the session list: any session created
+        // here would be established after close() has already torn everything down. Because each
+        // new session's bootstrap continuation strongly captures the session and this bucket (via
+        // on_configuration_update()/on_stop()), and nothing stops a session added after close(),
+        // it would strand the whole bucket/session graph as a pure cycle that LeakSanitizer reports
+        // as leaked. bootstrap() guards the initial session the same way (see closed_ there).
+        return;
+      }
       std::map<size_t, io::mcbp_session> new_sessions{};
 
       std::size_t next_index{ 0 };
@@ -848,12 +1092,11 @@ public:
         for (auto it = sessions_.begin(); it != sessions_.end(); ++it) {
           if (it->second.bootstrap_hostname() == hostname &&
               it->second.bootstrap_port_number() == port) {
-            CB_LOG_DEBUG(R"({} rev={}, preserve session="{}", address="{}:{}", index={}->{})",
+            CB_LOG_DEBUG(R"({} rev={}, preserve session="{}", address="{}", index={}->{})",
                          log_prefix_,
                          config.rev_str(),
                          it->second.id(),
-                         it->second.bootstrap_hostname(),
-                         it->second.bootstrap_port(),
+                         logger::system_data(it->second.bootstrap_address()),
                          it->first,
                          next_index);
             new_sessions.insert_or_assign(next_index, std::move(it->second));
@@ -885,23 +1128,21 @@ public:
                                known_features_)
             : io::mcbp_session(
                 client_id_, node.node_uuid, ctx_, origin, state_listener_, name_, known_features_);
-        CB_LOG_DEBUG(R"({} rev={}, add session="{}", address="{}:{}", index={})",
+        CB_LOG_DEBUG(R"({} rev={}, add session="{}", address="{}", index={})",
                      log_prefix_,
                      config.rev_str(),
                      session.id(),
-                     hostname,
-                     port,
+                     logger::system_data(fmt::format("{}:{}", hostname, port)),
                      node.index);
         session.bootstrap(
           [self = shared_from_this(), session, idx = next_index](
             std::error_code err, topology::configuration cfg) mutable {
             if (err) {
               CB_LOG_WARNING(
-                R"({} failed to bootstrap session="{}", address="{}:{}", index={}, ec={})",
+                R"({} failed to bootstrap session="{}", address="{}", index={}, ec={})",
                 session.log_prefix(),
                 session.id(),
-                session.bootstrap_hostname(),
-                session.bootstrap_port(),
+                logger::system_data(session.bootstrap_address()),
                 idx,
                 err.message());
               return self->remove_session(session.id());
@@ -920,18 +1161,24 @@ public:
       std::swap(sessions_, new_sessions);
 
       for (auto it = new_sessions.begin(); it != new_sessions.end(); ++it) {
-        CB_LOG_DEBUG(R"({} rev={}, drop session="{}", address="{}:{}", index={})",
+        CB_LOG_DEBUG(R"({} rev={}, drop session="{}", address="{}", index={})",
                      log_prefix_,
                      config.rev_str(),
                      it->second.id(),
-                     it->second.bootstrap_hostname(),
-                     it->second.bootstrap_port(),
+                     logger::system_data(it->second.bootstrap_address()),
                      it->first);
-        asio::post(asio::bind_executor(ctx_, [session = std::move(it->second)]() mutable {
-          return session.stop(retry_reason::do_not_retry);
-        }));
+        dropped_sessions.push_back(std::move(it->second));
       }
     }
+    // Stop dropped sessions synchronously, after releasing sessions_mutex_ (stop() fires on_stop ->
+    // remove_session(), which re-acquires the mutex). Stopping synchronously -- rather than via
+    // asio::post -- clears each session's bucket-capturing handlers (config_listeners_,
+    // on_stop_handler_) right now. A posted stop can be abandoned by io_context shutdown before it
+    // runs, leaving those handlers set and stranding the whole bucket/session graph as a pure cycle
+    // (LeakSanitizer reports it as all-indirect leaks).
+    for (auto& session : dropped_sessions) {
+      session.stop(retry_reason::do_not_retry);
+    }
   }
 
   [[nodiscard]] auto find_session_by_index(std::size_t index) const
@@ -944,14 +1191,15 @@ public:
     return {};
   }
 
-  [[nodiscard]] auto find_or_connect_session_by_index(std::size_t index)
-    -> std::optional<io::mcbp_session>
+  [[nodiscard]] auto find_or_connect_session_by_index(std::size_t index) -> routed_session
   {
     if (auto session = find_session_by_index(index); session) {
-      return session;
+      // A session exists, so it is safe to defer behind it. That holds even
+      // if it is still bootstrapping: the completion that finishes it is what
+      // drains the deferred queue.
+      return { session, true };
     }
-    connect_session(index);
-    return {};
+    return { {}, connect_session(index) };
   }
 
   [[nodiscard]] auto next_session_index() -> std::size_t
@@ -1048,9 +1296,18 @@ public:
 
   auto defer_command(utils::movable_function<void(std::error_code)> command) -> std::error_code
   {
-    const std::scoped_lock lock_for_deferred_commands(deferred_commands_mutex_);
-    deferred_commands_.emplace(std::move(command));
-    return {};
+    {
+      const std::scoped_lock lock_for_deferred_commands(deferred_commands_mutex_);
+      if (!closed_) {
+        deferred_commands_.emplace(std::move(command));
+        return {};
+      }
+    }
+    // The bucket was closed (and its deferred queue already drained) before this command could be
+    // parked. Enqueuing now would strand the command forever, leaking whatever its continuation
+    // captures. Complete it with a cancellation instead so its state is released (CXXCBC-846).
+    command(errc::common::request_canceled);
+    return errc::common::request_canceled;
   }
 
   void for_each_session(utils::movable_function<void(io::mcbp_session&)> handler)
@@ -1097,7 +1354,17 @@ private:
   std::queue<utils::movable_function<void(std::error_code)>> deferred_commands_{};
   std::mutex deferred_commands_mutex_{};
 
+  // Requests waiting for a retry backoff, so that close() can complete them.
+  mcbp::operation_queue retry_backoff_queue_{};
+
   std::map<size_t, io::mcbp_session> sessions_{};
+  // The very first session is only moved into sessions_ once it finishes bootstrapping; until then
+  // it lives solely inside its own bootstrap continuation (which captures a copy of the session and
+  // this bucket). If the cluster is closed while that bootstrap is in flight (e.g. a bucket that
+  // never opens), close() would otherwise never see the session, so its self-capture would strand
+  // the session, the bucket, and the whole cluster graph. Track it here so close() can stop it and
+  // let the continuation fire. Guarded by sessions_mutex_.
+  std::optional<io::mcbp_session> bootstrapping_session_{};
   mutable std::mutex sessions_mutex_{};
   std::atomic_size_t round_robin_next_{ 0 };
 };
@@ -1287,6 +1554,33 @@ bucket::config_rev() const -> std::string
   return impl_->config_rev();
 }
 
+auto
+bucket::config_snapshot() const -> std::shared_ptr<topology::configuration>
+{
+  return impl_->config_snapshot();
+}
+
+bucket_unit_test_api::bucket_unit_test_api(std::shared_ptr<bucket_impl> impl)
+  : impl_{ std::move(impl) }
+{
+}
+
+void
+bucket_unit_test_api::resolve_response(const std::shared_ptr<mcbp::queue_request>& request,
+                                       const std::shared_ptr<mcbp::queue_response>& response,
+                                       std::error_code ec,
+                                       retry_reason reason,
+                                       std::optional<key_value_error_map_info> error_info)
+{
+  impl_->resolve_response(request, response, ec, reason, std::move(error_info));
+}
+
+auto
+bucket::unit_test_api() -> bucket_unit_test_api
+{
+  return bucket_unit_test_api(impl_);
+}
+
 auto
 bucket::direct_dispatch(std::shared_ptr<mcbp::queue_request> req) -> std::error_code
 {
@@ -1300,8 +1594,8 @@ bucket::direct_re_queue(const std::shared_ptr<mcbp::queue_request>& req, bool is
   return impl_->direct_re_queue(req, is_retry);
 }
 
-void
-bucket::connect_session(std::size_t index)
+auto
+bucket::connect_session(std::size_t index) -> bool
 {
   return impl_->connect_session(index);
 }