couchbase 3.7.0 → 3.8.1

data/ext/couchbase/core/impl/collection.cxx

@@ -21,6 +21,8 @@
 #include "get_all_replicas.hxx"
 #include "get_any_replica.hxx"
 #include "internal_scan_result.hxx"
+#include "invoke_with_node_id.hxx"
+#include "observability_recorder.hxx"
 #include "observe_poll.hxx"
 
 #include "core/agent_group.hxx"
@@ -28,6 +30,7 @@
 #include "core/cluster.hxx"
 #include "core/impl/subdoc/command.hxx"
 #include "core/logger/logger.hxx"
+#include "core/metrics/meter_wrapper.hxx"
 #include "core/operations/document_append.hxx"
 #include "core/operations/document_decrement.hxx"
 #include "core/operations/document_exists.hxx"
@@ -53,6 +56,8 @@
 #include "core/range_scan_orchestrator.hxx"
 #include "core/range_scan_orchestrator_options.hxx"
 #include "core/topology/configuration.hxx"
+#include "core/tracing/constants.hxx"
+#include "core/tracing/tracer_wrapper.hxx"
 
 #include <couchbase/binary_collection.hxx>
 #include <couchbase/cas.hxx>
@@ -92,6 +97,10 @@
 
 #include <spdlog/fmt/bundled/core.h>
 
+#include <asio/error.hpp>
+#include <asio/steady_timer.hpp>
+
+#include <atomic>
 #include <chrono>
 #include <cstdint>
 #include <future>
@@ -103,8 +112,12 @@
 #include <variant>
 #include <vector>
 
+#include "core/tracing/attribute_helpers.hxx"
+
 namespace couchbase
 {
+using core::impl::invoke_with_node_id;
+
 class collection_impl : public std::enable_shared_from_this<collection_impl>
 {
 public:
@@ -148,47 +161,69 @@ public:
 
   void get(std::string document_key, get_options::built options, get_handler&& handler) const
   {
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_get, options.parent_span);
+
     if (!options.with_expiry && options.projections.empty()) {
-      return core_.execute(
-        core::operations::get_request{
-          core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-          {},
-          {},
-          options.timeout,
-          { options.retry_strategy },
-          options.parent_span,
+      core::operations::get_request request{
+        core::document_id{
+          bucket_name_,
+          scope_name_,
+          name_,
+          std::move(document_key),
         },
-        [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
-          return handler(
-            core::impl::make_error(std::move(resp.ctx)),
-            get_result{
-              resp.cas, { std::move(resp.value), resp.flags }, {}, std::move(crypto_manager) });
-        });
-    }
-    return core_.execute(
-      core::operations::get_projected_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
         {},
-        options.projections,
-        options.with_expiry,
         {},
-        false,
         options.timeout,
         { options.retry_strategy },
-        options.parent_span,
+        obs_rec->operation_span(),
+      };
+      return core_.execute(std::move(request),
+                           [obs_rec = std::move(obs_rec),
+                            crypto_manager = crypto_manager_,
+                            handler = std::move(handler)](auto resp) mutable {
+                             obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+                             invoke_with_node_id(std::move(handler),
+                                                 core::impl::make_error(std::move(resp.ctx)),
+                                                 get_result{ resp.cas,
+                                                             { std::move(resp.value), resp.flags },
+                                                             {},
+                                                             std::move(crypto_manager) });
+                           });
+    }
+    core::operations::get_projected_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
-        std::optional<std::chrono::system_clock::time_point> expiry_time{};
-        if (resp.expiry && resp.expiry.value() > 0) {
-          expiry_time.emplace(std::chrono::seconds{ resp.expiry.value() });
-        }
-        return handler(core::impl::make_error(std::move(resp.ctx)),
-                       get_result{ resp.cas,
-                                   { std::move(resp.value), resp.flags },
-                                   expiry_time,
-                                   std::move(crypto_manager) });
-      });
+      {},
+      {},
+      options.projections,
+      options.with_expiry,
+      {},
+      false,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    return core_.execute(std::move(request),
+                         [obs_rec = std::move(obs_rec),
+                          crypto_manager = crypto_manager_,
+                          handler = std::move(handler)](auto resp) mutable {
+                           std::optional<std::chrono::system_clock::time_point> expiry_time{};
+                           if (resp.expiry && resp.expiry.value() > 0) {
+                             expiry_time.emplace(std::chrono::seconds{ resp.expiry.value() });
+                           }
+                           obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+                           invoke_with_node_id(std::move(handler),
+                                               core::impl::make_error(std::move(resp.ctx)),
+                                               get_result{ resp.cas,
+                                                           { std::move(resp.value), resp.flags },
+                                                           expiry_time,
+                                                           std::move(crypto_manager) });
+                         });
   }
 
   void get_and_touch(std::string document_key,
@@ -196,22 +231,36 @@ public:
                      get_and_touch_options::built options,
                      get_and_touch_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::get_and_touch_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        expiry,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_and_touch,
+                                                 options.parent_span);
+
+    core::operations::get_and_touch_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
-        return handler(
-          core::impl::make_error(std::move(resp.ctx)),
-          get_result{
-            resp.cas, { std::move(resp.value), resp.flags }, {}, std::move(crypto_manager) });
-      });
+      {},
+      {},
+      expiry,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+
+    return core_.execute(std::move(request),
+                         [obs_rec = std::move(obs_rec),
+                          crypto_manager = crypto_manager_,
+                          handler = std::move(handler)](auto resp) mutable {
+                           obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+                           invoke_with_node_id(std::move(handler),
+                                               core::impl::make_error(std::move(resp.ctx)),
+                                               get_result{ resp.cas,
+                                                           { std::move(resp.value), resp.flags },
+                                                           {},
+                                                           std::move(crypto_manager) });
+                         });
   }
 
   void touch(std::string document_key,
@@ -219,18 +268,29 @@ public:
              touch_options::built options,
              touch_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::touch_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        expiry,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_touch, options.parent_span);
+
+    core::operations::touch_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](const auto& resp) mutable {
-        return handler(core::impl::make_error(std::move(resp.ctx)), result{ resp.cas });
+      {},
+      {},
+      expiry,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](const auto& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+        invoke_with_node_id(
+          std::move(handler), core::impl::make_error(std::move(resp.ctx)), result{ resp.cas });
       });
   }
 
@@ -238,51 +298,82 @@ public:
                        const get_any_replica_options::built& options,
                        core::impl::movable_get_any_replica_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::get_any_replica_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        options.timeout,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_any_replica,
+                                                 options.parent_span);
+
+    core::operations::get_any_replica_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
-        return handler(core::impl::make_error(std::move(resp.ctx)),
-                       get_replica_result{
-                         resp.cas,
-                         resp.replica,
-                         { std::move(resp.value), resp.flags },
-                         std::move(crypto_manager),
-                       });
-      });
+      options.timeout,
+      options.read_preference,
+      obs_rec->operation_span(),
+    };
+    return core_.execute(std::move(request),
+                         [obs_rec = std::move(obs_rec),
+                          crypto_manager = crypto_manager_,
+                          handler = std::move(handler)](auto resp) mutable {
+                           obs_rec->finish(resp.ctx.ec());
+                           invoke_with_node_id(std::move(handler),
+                                               core::impl::make_error(std::move(resp.ctx)),
+                                               get_replica_result{
+                                                 resp.cas,
+                                                 resp.replica,
+                                                 { std::move(resp.value), resp.flags },
+                                                 std::move(crypto_manager),
+                                               });
+                         });
   }
 
   void get_all_replicas(std::string document_key,
                         const get_all_replicas_options::built& options,
                         core::impl::movable_get_all_replicas_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::get_all_replicas_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        options.timeout,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_all_replicas,
+                                                 options.parent_span);
+
+    core::operations::get_all_replicas_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
-        get_all_replicas_result result{};
-        for (auto& entry : resp.entries) {
-          result.emplace_back(get_replica_result{
-            entry.cas,
-            entry.replica,
-            { std::move(entry.value), entry.flags },
-            crypto_manager,
-          });
-        }
-        return handler(core::impl::make_error(std::move(resp.ctx)), std::move(result));
-      });
+      options.timeout,
+      options.read_preference,
+      obs_rec->operation_span(),
+    };
+    return core_.execute(std::move(request),
+                         [obs_rec = std::move(obs_rec),
+                          crypto_manager = crypto_manager_,
+                          handler = std::move(handler)](auto resp) mutable {
+                           get_all_replicas_result result{};
+                           for (auto& entry : resp.entries) {
+                             get_replica_result replica_result{
+                               entry.cas,
+                               entry.replica,
+                               { std::move(entry.value), entry.flags },
+                               crypto_manager,
+                             };
+                             replica_result.node_id(std::move(entry.dispatched_to_node_id));
+                             result.emplace_back(std::move(replica_result));
+                           }
+                           obs_rec->finish(resp.ctx.ec());
+                           return handler(core::impl::make_error(std::move(resp.ctx)),
+                                          std::move(result));
+                         });
   }
 
   void remove(std::string document_key,
               remove_options::built options,
               remove_handler&& handler) const
   {
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_remove, options.parent_span);
+
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -290,23 +381,28 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::remove_request request{
+        std::move(id),
+        {},
+        {},
+        options.cas,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::remove_request{
-          std::move(id),
-          {},
-          {},
-          options.cas,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
-            return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            invoke_with_node_id(
+              std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            return;
           }
-          return handler(core::impl::make_error(std::move(resp.ctx)),
-                         mutation_result{ resp.cas, std::move(resp.token) });
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
         });
     }
 
@@ -318,33 +414,44 @@ public:
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
-      options.parent_span,
+      obs_rec->operation_span(),
     };
-    return core_.execute(std::move(request),
-                         [core = core_, id = std::move(id), options, handler = std::move(handler)](
-                           auto&& resp) mutable {
-                           if (resp.ctx.ec()) {
-                             return handler(core::impl::make_error(std::move(resp.ctx)),
-                                            mutation_result{ resp.cas, std::move(resp.token) });
-                           }
-                           auto token = resp.token;
-                           core::impl::initiate_observe_poll(
-                             core,
-                             std::move(id),
-                             token,
-                             options.timeout,
-                             options.persist_to,
-                             options.replicate_to,
-                             [resp, handler = std::move(handler)](std::error_code ec) mutable {
-                               if (ec) {
-                                 resp.ctx.override_ec(ec);
-                                 return handler(core::impl::make_error(std::move(resp.ctx)),
-                                                mutation_result{});
-                               }
-                               return handler(core::impl::make_error(std::move(resp.ctx)),
-                                              mutation_result{ resp.cas, std::move(resp.token) });
-                             });
-                         });
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto&& resp) mutable {
+        if (resp.ctx.ec()) {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
+          return;
+        }
+        auto token = resp.token;
+        core::impl::initiate_observe_poll(
+          core,
+          std::move(id),
+          token,
+          options.timeout,
+          options.persist_to,
+          options.replicate_to,
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
+            if (ec) {
+              resp.ctx.override_ec(ec);
+              invoke_with_node_id(
+                std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+              return;
+            }
+            invoke_with_node_id(std::move(handler),
+                                core::impl::make_error(std::move(resp.ctx)),
+                                mutation_result{ resp.cas, std::move(resp.token) });
+          });
+      });
   }
 
   void get_and_lock(std::string document_key,
@@ -352,22 +459,35 @@ public:
                     get_and_lock_options::built options,
                     get_and_lock_handler&& handler) const
   {
-    core_.execute(
-      core::operations::get_and_lock_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        static_cast<uint32_t>(lock_duration.count()),
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_and_lock,
+                                                 options.parent_span);
+
+    core::operations::get_and_lock_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto&& resp) mutable {
-        return handler(
-          core::impl::make_error(std::move(resp.ctx)),
-          get_result{
-            resp.cas, { std::move(resp.value), resp.flags }, {}, std::move(crypto_manager) });
-      });
+      {},
+      {},
+      static_cast<uint32_t>(lock_duration.count()),
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    core_.execute(std::move(request),
+                  [obs_rec = std::move(obs_rec),
+                   crypto_manager = crypto_manager_,
+                   handler = std::move(handler)](auto&& resp) mutable {
+                    obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+                    invoke_with_node_id(std::move(handler),
+                                        core::impl::make_error(std::move(resp.ctx)),
+                                        get_result{ resp.cas,
+                                                    { std::move(resp.value), resp.flags },
+                                                    {},
+                                                    std::move(crypto_manager) });
+                  });
   }
 
   void unlock(std::string document_key,
@@ -375,17 +495,27 @@ public:
               unlock_options::built options,
               unlock_handler&& handler) const
   {
-    core_.execute(
-      core::operations::unlock_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        cas,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_unlock, options.parent_span);
+
+    core::operations::unlock_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto&& resp) mutable {
+      {},
+      {},
+      cas,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto&& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)));
       });
   }
@@ -394,18 +524,29 @@ public:
               exists_options::built options,
               exists_handler&& handler) const
   {
-    core_.execute(
-      core::operations::exists_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_exists, options.parent_span);
+
+    core::operations::exists_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto&& resp) mutable {
-        return handler(core::impl::make_error(std::move(resp.ctx)),
-                       exists_result{ resp.cas, resp.exists() });
+      {},
+      {},
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto&& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+        invoke_with_node_id(std::move(handler),
+                            core::impl::make_error(std::move(resp.ctx)),
+                            exists_result{ resp.cas, resp.exists() });
       });
   }
 
@@ -414,25 +555,33 @@ public:
                  lookup_in_options::built options,
                  lookup_in_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::lookup_in_request{
-        core::document_id{
-          bucket_name_,
-          scope_name_,
-          name_,
-          std::move(document_key),
-        },
-        {},
-        {},
-        options.access_deleted,
-        specs,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_lookup_in, options.parent_span);
+
+    core::operations::lookup_in_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto resp) mutable {
+      {},
+      {},
+      options.access_deleted,
+      specs,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+
         if (resp.ctx.ec()) {
-          return handler(core::impl::make_error(std::move(resp.ctx)), lookup_in_result{});
+          invoke_with_node_id(
+            std::move(handler), core::impl::make_error(std::move(resp.ctx)), lookup_in_result{});
+          return;
         }
 
         std::vector<lookup_in_result::entry> entries{};
@@ -446,8 +595,9 @@ public:
             entry.ec,
           });
         }
-        return handler(core::impl::make_error(std::move(resp.ctx)),
-                       lookup_in_result{ resp.cas, std::move(entries), resp.deleted });
+        invoke_with_node_id(std::move(handler),
+                            core::impl::make_error(std::move(resp.ctx)),
+                            lookup_in_result{ resp.cas, std::move(entries), resp.deleted });
       });
   }
 
@@ -456,15 +606,24 @@ public:
                               const lookup_in_all_replicas_options::built& options,
                               lookup_in_all_replicas_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::lookup_in_all_replicas_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        specs,
-        options.timeout,
-        options.parent_span,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_lookup_in_all_replicas, options.parent_span);
+
+    core::operations::lookup_in_all_replicas_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto resp) mutable {
+      specs,
+      options.timeout,
+      obs_rec->operation_span(),
+      options.read_preference,
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
         lookup_in_all_replicas_result result{};
         for (auto& res : resp.entries) {
           std::vector<lookup_in_result::entry> entries;
@@ -478,14 +637,17 @@ public:
               field.ec,
             });
           }
-          result.emplace_back(lookup_in_replica_result{
+          lookup_in_replica_result replica_result{
             res.cas,
             std::move(entries),
             res.deleted,
             res.is_replica,
-          });
+          };
+          replica_result.node_id(std::move(res.dispatched_to_node_id));
+          result.emplace_back(std::move(replica_result));
         }
-        return handler(core::impl::make_error(std::move(resp.ctx)), result);
+        obs_rec->finish(resp.ctx.ec());
+        return handler(core::impl::make_error(std::move(resp.ctx)), std::move(result));
       });
   }
 
@@ -494,15 +656,24 @@ public:
                              const lookup_in_any_replica_options::built& options,
                              lookup_in_any_replica_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::lookup_in_any_replica_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        specs,
-        options.timeout,
-        options.parent_span,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_lookup_in_any_replica, options.parent_span);
+
+    core::operations::lookup_in_any_replica_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto resp) mutable {
+      specs,
+      options.timeout,
+      obs_rec->operation_span(),
+      options.read_preference,
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
         std::vector<lookup_in_result::entry> entries;
         entries.reserve(resp.fields.size());
         for (auto& field : resp.fields) {
@@ -514,8 +685,9 @@ public:
             field.ec,
           });
         }
-        entries.reserve(resp.fields.size());
-        return handler(
+        obs_rec->finish(resp.ctx.ec());
+        invoke_with_node_id(
+          std::move(handler),
           core::impl::make_error(std::move(resp.ctx)),
           lookup_in_replica_result{ resp.cas, std::move(entries), resp.deleted, resp.is_replica });
       });
@@ -526,6 +698,9 @@ public:
                  mutate_in_options::built options,
                  mutate_in_handler&& handler) const
   {
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_mutate_in, options.parent_span, options.durability_level);
+
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -533,27 +708,31 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::mutate_in_request request{
+        std::move(id),
+        {},
+        {},
+        options.cas,
+        options.access_deleted,
+        options.create_as_deleted,
+        false,
+        options.expiry,
+        options.store_semantics,
+        specs,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        options.preserve_expiry,
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::mutate_in_request{
-          std::move(id),
-          {},
-          {},
-          options.cas,
-          options.access_deleted,
-          options.create_as_deleted,
-          false,
-          options.expiry,
-          options.store_semantics,
-          specs,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.preserve_expiry,
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
-            return handler(core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
+            invoke_with_node_id(
+              std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
+            return;
           }
           std::vector<mutate_in_result::entry> entries{};
           entries.reserve(resp.fields.size());
@@ -564,7 +743,8 @@ public:
               entry.original_index,
             });
           }
-          return handler(
+          invoke_with_node_id(
+            std::move(handler),
             core::impl::make_error(std::move(resp.ctx)),
             mutate_in_result{ resp.cas, std::move(resp.token), std::move(entries), resp.deleted });
         });
@@ -585,51 +765,68 @@ public:
       options.timeout,
       { options.retry_strategy },
       options.preserve_expiry,
-      options.parent_span,
+      obs_rec->operation_span(),
     };
-    return core_.execute(
-      std::move(request),
-      [core = core_, id = std::move(id), options, handler = std::move(handler)](
-        auto&& resp) mutable {
-        if (resp.ctx.ec()) {
-          return handler(core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
-        }
+    return core_.execute(std::move(request),
+                         [obs_rec = std::move(obs_rec),
+                          core = core_,
+                          id = std::move(id),
+                          options,
+                          handler = std::move(handler)](auto&& resp) mutable {
+                           if (resp.ctx.ec()) {
+                             obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+                             invoke_with_node_id(std::move(handler),
+                                                 core::impl::make_error(std::move(resp.ctx)),
+                                                 mutate_in_result{});
+                             return;
+                           }
 
-        auto token = resp.token;
-        core::impl::initiate_observe_poll(
-          core,
-          std::move(id),
-          token,
-          options.timeout,
-          options.persist_to,
-          options.replicate_to,
-          [resp, handler = std::move(handler)](std::error_code ec) mutable {
-            if (ec) {
-              resp.ctx.override_ec(ec);
-              return handler(core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
-            }
-            std::vector<mutate_in_result::entry> entries{};
-            entries.reserve(resp.fields.size());
-            for (auto& entry : resp.fields) {
-              entries.emplace_back(mutate_in_result::entry{
-                std::move(entry.path),
-                std::move(entry.value),
-                entry.original_index,
-              });
-            }
-            return handler(core::impl::make_error(std::move(resp.ctx)),
-                           mutate_in_result{
-                             resp.cas, std::move(resp.token), std::move(entries), resp.deleted });
-          });
-      });
+                           auto token = resp.token;
+                           core::impl::initiate_observe_poll(
+                             core,
+                             std::move(id),
+                             token,
+                             options.timeout,
+                             options.persist_to,
+                             options.replicate_to,
+                             [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+                               std::error_code ec) mutable {
+                               obs_rec->finish(resp.ctx.retry_attempts(), ec);
+                               if (ec) {
+                                 resp.ctx.override_ec(ec);
+                                 invoke_with_node_id(std::move(handler),
+                                                     core::impl::make_error(std::move(resp.ctx)),
+                                                     mutate_in_result{});
+                                 return;
+                               }
+                               std::vector<mutate_in_result::entry> entries{};
+                               entries.reserve(resp.fields.size());
+                               for (auto& entry : resp.fields) {
+                                 entries.emplace_back(mutate_in_result::entry{
+                                   std::move(entry.path),
+                                   std::move(entry.value),
+                                   entry.original_index,
+                                 });
+                               }
+                               invoke_with_node_id(std::move(handler),
+                                                   core::impl::make_error(std::move(resp.ctx)),
+                                                   mutate_in_result{ resp.cas,
+                                                                     std::move(resp.token),
+                                                                     std::move(entries),
+                                                                     resp.deleted });
+                             });
+                         });
   }
 
   void upsert(std::string document_key,
-              codec::encoded_value encoded,
+              std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
               upsert_options::built options,
               upsert_handler&& handler) const
   {
-    auto value = std::move(encoded);
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_upsert, options.parent_span, options.durability_level);
+
+    auto [data, flags] = get_encoded_value(std::move(value), obs_rec);
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -637,45 +834,55 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::upsert_request request{
+        std::move(id),
+        std::move(data),
+        {},
+        {},
+        flags,
+        options.expiry,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        options.preserve_expiry,
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::upsert_request{
-          std::move(id),
-          std::move(value.data),
-          {},
-          {},
-          value.flags,
-          options.expiry,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.preserve_expiry,
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
-          return handler(core::impl::make_error(std::move(resp.ctx)),
-                         mutation_result{ resp.cas, std::move(resp.token) });
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
         });
     }
 
     core::operations::upsert_request request{
       id,
-      std::move(value.data),
+      std::move(data),
       {},
       {},
-      value.flags,
+      flags,
       options.expiry,
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
       options.preserve_expiry,
-      options.parent_span,
+      obs_rec->operation_span(),
     };
     return core_.execute(
       std::move(request),
-      [core = core_, id = std::move(id), options, handler = std::move(handler)](auto resp) mutable {
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto resp) mutable {
         if (resp.ctx.ec()) {
-          return handler(core::impl::make_error(std::move(resp.ctx)),
-                         mutation_result{ resp.cas, std::move(resp.token) });
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
+          return;
         }
 
         auto token = resp.token;
@@ -686,23 +893,31 @@ public:
           options.timeout,
           options.persist_to,
           options.replicate_to,
-          [resp, handler = std::move(handler)](std::error_code ec) mutable {
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
             if (ec) {
               resp.ctx.override_ec(ec);
-              return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+              invoke_with_node_id(
+                std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+              return;
             }
-            return handler(core::impl::make_error(std::move(resp.ctx)),
-                           mutation_result{ resp.cas, std::move(resp.token) });
+            invoke_with_node_id(std::move(handler),
+                                core::impl::make_error(std::move(resp.ctx)),
+                                mutation_result{ resp.cas, std::move(resp.token) });
           });
       });
   }
 
   void insert(std::string document_key,
-              codec::encoded_value encoded,
+              std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
               insert_options::built options,
               insert_handler&& handler) const
   {
-    auto value = std::move(encoded);
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_insert, options.parent_span, options.durability_level);
+
+    auto [data, flags] = get_encoded_value(std::move(value), obs_rec);
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -710,46 +925,58 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::insert_request request{
+        std::move(id),
+        std::move(data),
+        {},
+        {},
+        flags,
+        options.expiry,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::insert_request{
-          std::move(id),
-          std::move(value.data),
-          {},
-          {},
-          value.flags,
-          options.expiry,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto&& resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto&& resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
-            return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            invoke_with_node_id(
+              std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            return;
           }
-          return handler(core::impl::make_error(std::move(resp.ctx)),
-                         mutation_result{ resp.cas, std::move(resp.token) });
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
         });
     }
 
     core::operations::insert_request request{
       id,
-      std::move(value.data),
+      std::move(data),
       {},
       {},
-      value.flags,
+      flags,
       options.expiry,
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
-      options.parent_span,
+      obs_rec->operation_span(),
     };
     return core_.execute(
       std::move(request),
-      [core = core_, id = std::move(id), options, handler = std::move(handler)](auto resp) mutable {
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto resp) mutable {
         if (resp.ctx.ec()) {
-          return handler(core::impl::make_error(std::move(resp.ctx)),
-                         mutation_result{ resp.cas, std::move(resp.token) });
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
+          return;
         }
 
         auto token = resp.token;
@@ -760,22 +987,32 @@ public:
           options.timeout,
           options.persist_to,
           options.replicate_to,
-          [resp, handler = std::move(handler)](std::error_code ec) mutable {
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
             if (ec) {
               resp.ctx.override_ec(ec);
-              return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+              invoke_with_node_id(
+                std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+              return;
             }
-            return handler(core::impl::make_error(std::move(resp.ctx)),
-                           mutation_result{ resp.cas, std::move(resp.token) });
+            invoke_with_node_id(std::move(handler),
+                                core::impl::make_error(std::move(resp.ctx)),
+                                mutation_result{ resp.cas, std::move(resp.token) });
           });
       });
   }
+
   void replace(std::string document_key,
-               codec::encoded_value encoded,
+               std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
                replace_options::built options,
                replace_handler&& handler) const
   {
-    auto value = std::move(encoded);
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_replace, options.parent_span, options.durability_level);
+
+    auto [data, flags] = get_encoded_value(std::move(value), obs_rec);
+
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -783,74 +1020,93 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::replace_request request{
+        std::move(id),
+        std::move(data),
+        {},
+        {},
+        flags,
+        options.expiry,
+        options.cas,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        options.preserve_expiry,
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::replace_request{
-          std::move(id),
-          std::move(value.data),
-          {},
-          {},
-          value.flags,
-          options.expiry,
-          options.cas,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.preserve_expiry,
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
-            return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            invoke_with_node_id(
+              std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            return;
           }
-          return handler(core::impl::make_error(std::move(resp.ctx)),
-                         mutation_result{ resp.cas, std::move(resp.token) });
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
         });
     }
 
     core::operations::replace_request request{
       id,
-      std::move(value.data),
+      std::move(data),
       {},
       {},
-      value.flags,
+      flags,
       options.expiry,
       options.cas,
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
       options.preserve_expiry,
-      options.parent_span,
+      obs_rec->operation_span(),
     };
-    return core_.execute(std::move(request),
-                         [core = core_, id = std::move(id), options, handler = std::move(handler)](
-                           auto&& resp) mutable {
-                           if (resp.ctx.ec()) {
-                             return handler(core::impl::make_error(std::move(resp.ctx)),
-                                            mutation_result{ resp.cas, std::move(resp.token) });
-                           }
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto&& resp) mutable {
+        if (resp.ctx.ec()) {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+          invoke_with_node_id(std::move(handler),
+                              core::impl::make_error(std::move(resp.ctx)),
+                              mutation_result{ resp.cas, std::move(resp.token) });
+          return;
+        }
 
-                           auto token = resp.token;
-                           core::impl::initiate_observe_poll(
-                             core,
-                             std::move(id),
-                             token,
-                             options.timeout,
-                             options.persist_to,
-                             options.replicate_to,
-                             [resp, handler = std::move(handler)](std::error_code ec) mutable {
-                               if (ec) {
-                                 resp.ctx.override_ec(ec);
-                                 return handler(core::impl::make_error(std::move(resp.ctx)),
-                                                mutation_result{});
-                               }
-                               return handler(core::impl::make_error(std::move(resp.ctx)),
-                                              mutation_result{ resp.cas, std::move(resp.token) });
-                             });
-                         });
+        auto token = resp.token;
+        core::impl::initiate_observe_poll(
+          core,
+          std::move(id),
+          token,
+          options.timeout,
+          options.persist_to,
+          options.replicate_to,
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
+            if (ec) {
+              resp.ctx.override_ec(ec);
+              invoke_with_node_id(
+                std::move(handler), core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+              return;
+            }
+            invoke_with_node_id(std::move(handler),
+                                core::impl::make_error(std::move(resp.ctx)),
+                                mutation_result{ resp.cas, std::move(resp.token) });
+          });
+      });
   }
 
   void scan(scan_type::built scan_type, scan_options::built options, scan_handler&& handler) const
   {
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_scan, options.parent_span);
+
     core::range_scan_orchestrator_options orchestrator_opts{ options.ids_only };
     if (!options.mutation_state.empty()) {
       orchestrator_opts.consistent_with = core::mutation_state{ options.mutation_state };
@@ -903,22 +1159,31 @@ public:
 
     return core_.open_bucket(
       bucket_name_,
-      [this, handler = std::move(handler), orchestrator_opts, core_scan_type](
-        std::error_code ec) mutable {
+      [this,
+       obs_rec = std::move(obs_rec),
+       handler = std::move(handler),
+       orchestrator_opts,
+       core_scan_type](std::error_code ec) mutable {
         if (ec) {
+          obs_rec->finish(ec);
           return handler(error(ec), {});
         }
         return core_.with_bucket_configuration(
           bucket_name_,
-          [this, handler = std::move(handler), orchestrator_opts, core_scan_type](
-            std::error_code ec,
-            const std::shared_ptr<core::topology::configuration>& config) mutable {
+          [this,
+           obs_rec = std::move(obs_rec),
+           handler = std::move(handler),
+           orchestrator_opts,
+           core_scan_type](std::error_code ec,
+                           const std::shared_ptr<core::topology::configuration>& config) mutable {
             if (ec) {
+              obs_rec->finish(ec);
               return handler(
                 error(ec, "An error occurred when attempting to fetch the bucket configuration."),
                 {});
             }
             if (!config->capabilities.supports_range_scan()) {
+              obs_rec->finish(ec);
               return handler(error(errc::common::feature_not_available,
                                    "This bucket does not support range scan."),
                              {});
@@ -927,6 +1192,7 @@ public:
               core::agent_group(core_.io_context(), core::agent_group_config{ { core_ } });
             ec = agent_group.open_bucket(bucket_name_);
             if (ec) {
+              obs_rec->finish(ec);
               return handler(error(ec,
                                    fmt::format("An error occurred while opening the `{}` bucket.",
                                                bucket_name_)),
@@ -934,6 +1200,7 @@ public:
             }
             auto agent = agent_group.get_agent(bucket_name_);
             if (!agent.has_value()) {
+              obs_rec->finish(ec);
               return handler(
                 error(agent.error(),
                       fmt::format(
@@ -944,6 +1211,7 @@ public:
             if (!config->vbmap.has_value() || config->vbmap->empty()) {
               CB_LOG_WARNING("Unable to get vbucket map for `{}` - cannot perform scan operation",
                              bucket_name_);
+              obs_rec->finish(ec);
               return handler(error(errc::common::request_canceled,
                                    "No vbucket map included with the bucket config"),
                              {});
@@ -957,8 +1225,10 @@ public:
                                                               core_scan_type,
                                                               orchestrator_opts);
             return orchestrator.scan(
-              [crypto_manager = crypto_manager_,
+              [obs_rec = std::move(obs_rec),
+               crypto_manager = crypto_manager_,
                handler = std::move(handler)](auto ec, auto core_scan_result) mutable {
+                obs_rec->finish(ec);
                 if (ec) {
                   return handler(error(ec, "Error while starting the range scan"), {});
                 }
@@ -970,7 +1240,168 @@ public:
       });
   }
 
+  void node_id_for(std::string document_key,
+                   const node_id_for_options::built& options,
+                   node_id_for_handler&& handler) const
+  {
+    auto [origin_ec, origin] = core_.origin();
+    if (origin_ec) {
+      return handler(error{ origin_ec }, node_id{});
+    }
+    const bool is_tls = origin.options().enable_tls;
+    const auto timeout = options.timeout.value_or(origin.options().key_value_timeout);
+
+    // Shared state so that only one of {timeout timer, bucket-config callback}
+    // can invoke the user handler. with_bucket_configuration() is not
+    // cancellable — if config never arrives, the timer is the only way to
+    // guarantee the handler completes.
+    struct state {
+      std::atomic<bool> done{ false };
+      asio::steady_timer timer;
+      node_id_for_handler handler;
+
+      state(asio::io_context& ioc, node_id_for_handler&& h)
+        : timer{ ioc }
+        , handler{ std::move(h) }
+      {
+      }
+    };
+    auto shared = std::make_shared<state>(core_.io_context(), std::move(handler));
+
+    shared->timer.expires_after(timeout);
+    shared->timer.async_wait([shared](std::error_code timer_ec) -> void {
+      if (timer_ec == asio::error::operation_aborted) {
+        return;
+      }
+      if (shared->done.exchange(true)) {
+        return;
+      }
+      shared->handler(error{ errc::common::unambiguous_timeout }, node_id{});
+    });
+
+    core_.with_bucket_configuration(
+      bucket_name_,
+      [shared, document_key = std::move(document_key), is_tls](
+        std::error_code ec, const std::shared_ptr<core::topology::configuration>& config) -> void {
+        if (shared->done.exchange(true)) {
+          return;
+        }
+        shared->timer.cancel();
+        if (ec) {
+          return shared->handler(error{ ec }, node_id{});
+        }
+        if (!config || !config->vbmap.has_value() || config->vbmap->empty()) {
+          return shared->handler(error{ errc::network::configuration_not_available }, node_id{});
+        }
+        // Only server_index is needed here; vbucket is intentionally unused.
+        // Binding it as [[maybe_unused]] would require C++20 structured-
+        // binding attribute support, so take the pair by value and pull the
+        // field we need instead.
+        const auto mapping = config->map_key(document_key, 0);
+        const auto& server_index = mapping.second;
+        if (!server_index.has_value() || server_index.value() >= config->nodes.size()) {
+          // An unresolvable server_index means the vBucket map is present
+          // but does not map this key to a known node (out-of-range or
+          // missing entry). Surface this as configuration_not_available
+          // rather than request_canceled so callers can distinguish an
+          // unusable map from an actual caller-initiated cancellation.
+          return shared->handler(error{ errc::network::configuration_not_available }, node_id{});
+        }
+        return shared->handler({}, config->nodes[server_index.value()].effective_node_id(is_tls));
+      });
+  }
+
+  void node_ids(const node_ids_options::built& options, node_ids_handler&& handler) const
+  {
+    auto [origin_ec, origin] = core_.origin();
+    if (origin_ec) {
+      return handler(error{ origin_ec }, {});
+    }
+    const bool is_tls = origin.options().enable_tls;
+    const auto timeout = options.timeout.value_or(origin.options().key_value_timeout);
+
+    // Same shape as node_id_for: with_bucket_configuration() is not
+    // cancellable, so a steady_timer guarded by an atomic done-flag is
+    // the only way to bound the wait when no config arrives.
+    struct state {
+      std::atomic<bool> done{ false };
+      asio::steady_timer timer;
+      node_ids_handler handler;
+
+      state(asio::io_context& ioc, node_ids_handler&& h)
+        : timer{ ioc }
+        , handler{ std::move(h) }
+      {
+      }
+    };
+    auto shared = std::make_shared<state>(core_.io_context(), std::move(handler));
+
+    shared->timer.expires_after(timeout);
+    shared->timer.async_wait([shared](std::error_code timer_ec) -> void {
+      if (timer_ec == asio::error::operation_aborted) {
+        return;
+      }
+      if (shared->done.exchange(true)) {
+        return;
+      }
+      shared->handler(error{ errc::common::unambiguous_timeout }, {});
+    });
+
+    core_.with_bucket_configuration(
+      bucket_name_,
+      [shared, is_tls](std::error_code ec,
+                       const std::shared_ptr<core::topology::configuration>& config) -> void {
+        if (shared->done.exchange(true)) {
+          return;
+        }
+        shared->timer.cancel();
+        if (ec) {
+          return shared->handler(error{ ec }, {});
+        }
+        if (!config) {
+          // No config arrived but no error either — this should not
+          // happen in practice, but surface it the same way node_id_for
+          // does so callers can distinguish from request_canceled.
+          return shared->handler(error{ errc::network::configuration_not_available }, {});
+        }
+        return shared->handler({}, config->effective_node_ids(is_tls));
+      });
+  }
+
 private:
+  static auto get_encoded_value(
+    std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
+    const std::unique_ptr<core::impl::observability_recorder>& obs_rec) -> codec::encoded_value
+  {
+    if (std::holds_alternative<codec::encoded_value>(value)) {
+      return std::get<codec::encoded_value>(value);
+    }
+    const auto request_encoding_span = obs_rec->create_request_encoding_span();
+    auto encoded = std::get<std::function<codec::encoded_value()>>(value)();
+    request_encoding_span->end();
+    return encoded;
+  }
+
+  [[nodiscard]] auto create_observability_recorder(
+    const std::string& operation_name,
+    const std::shared_ptr<tracing::request_span>& parent_span,
+    const std::optional<durability_level> durability = {}) const
+    -> std::unique_ptr<core::impl::observability_recorder>
+  {
+    auto rec = core::impl::observability_recorder::create(
+      operation_name, parent_span, core_.tracer(), core_.meter());
+
+    rec->with_service(core::tracing::service::key_value);
+    rec->with_bucket_name(bucket_name_);
+    rec->with_scope_name(scope_name_);
+    rec->with_collection_name(name_);
+    if (durability.has_value()) {
+      rec->with_durability(durability.value());
+    }
+
+    return rec;
+  }
+
   core::cluster core_;
   std::string bucket_name_;
   std::string scope_name_;
@@ -1027,6 +1458,44 @@ collection::crypto_manager() const -> const std::shared_ptr<crypto::manager>&
   return impl_->crypto_manager();
 }
 
+void
+collection::node_id_for(std::string document_id,
+                        const node_id_for_options& options,
+                        node_id_for_handler&& handler) const
+{
+  return impl_->node_id_for(std::move(document_id), options.build(), std::move(handler));
+}
+
+auto
+collection::node_id_for(std::string document_id, const node_id_for_options& options) const
+  -> std::future<std::pair<error, node_id>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, node_id>>>();
+  auto future = barrier->get_future();
+  node_id_for(std::move(document_id), options, [barrier](auto err, auto nid) {
+    barrier->set_value({ std::move(err), std::move(nid) });
+  });
+  return future;
+}
+
+void
+collection::node_ids(const node_ids_options& options, node_ids_handler&& handler) const
+{
+  return impl_->node_ids(options.build(), std::move(handler));
+}
+
+auto
+collection::node_ids(const node_ids_options& options) const
+  -> std::future<std::pair<error, std::vector<node_id>>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, std::vector<node_id>>>>();
+  auto future = barrier->get_future();
+  node_ids(options, [barrier](auto err, auto nids) {
+    barrier->set_value({ std::move(err), std::move(nids) });
+  });
+  return future;
+}
+
 void
 collection::get(std::string document_id, const get_options& options, get_handler&& handler) const
 {
@@ -1392,6 +1861,30 @@ collection::upsert(std::string document_id,
   return future;
 }
 
+auto
+collection::upsert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const upsert_options& options,
+                   upsert_handler&& handler) const -> void
+{
+  impl_->upsert(
+    std::move(document_id), std::move(document_fn), options.build(), std::move(handler));
+}
+
+auto
+collection::upsert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const upsert_options& options) const
+  -> std::future<std::pair<error, mutation_result>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, mutation_result>>>();
+  auto future = barrier->get_future();
+  upsert(std::move(document_id), std::move(document_fn), options, [barrier](auto err, auto result) {
+    barrier->set_value({ std::move(err), std::move(result) });
+  });
+  return future;
+}
+
 void
 collection::insert(std::string document_id,
                    codec::encoded_value document,
@@ -1416,6 +1909,30 @@ collection::insert(std::string document_id,
   return future;
 }
 
+auto
+collection::insert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const insert_options& options,
+                   insert_handler&& handler) const -> void
+{
+  impl_->insert(
+    std::move(document_id), std::move(document_fn), options.build(), std::move(handler));
+}
+
+auto
+collection::insert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const insert_options& options) const
+  -> std::future<std::pair<error, mutation_result>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, mutation_result>>>();
+  auto future = barrier->get_future();
+  insert(std::move(document_id), std::move(document_fn), options, [barrier](auto err, auto result) {
+    barrier->set_value({ std::move(err), std::move(result) });
+  });
+  return future;
+}
+
 void
 collection::replace(std::string document_id,
                     codec::encoded_value document,
@@ -1440,6 +1957,31 @@ collection::replace(std::string document_id,
   return future;
 }
 
+void
+collection::replace(std::string document_id,
+                    std::function<codec::encoded_value()> document_fn,
+                    const replace_options& options,
+                    replace_handler&& handler) const
+{
+  impl_->replace(
+    std::move(document_id), std::move(document_fn), options.build(), std::move(handler));
+}
+
+auto
+collection::replace(std::string document_id,
+                    std::function<codec::encoded_value()> document_fn,
+                    const replace_options& options) const
+  -> std::future<std::pair<error, mutation_result>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, mutation_result>>>();
+  auto future = barrier->get_future();
+  replace(
+    std::move(document_id), std::move(document_fn), options, [barrier](auto err, auto result) {
+      barrier->set_value({ std::move(err), std::move(result) });
+    });
+  return future;
+}
+
 void
 collection::scan(const couchbase::scan_type& scan_type,
                  const couchbase::scan_options& options,