couchbase 3.7.0 → 3.8.0

data/ext/couchbase/core/impl/collection.cxx

@@ -21,6 +21,7 @@
 #include "get_all_replicas.hxx"
 #include "get_any_replica.hxx"
 #include "internal_scan_result.hxx"
+#include "observability_recorder.hxx"
 #include "observe_poll.hxx"
 
 #include "core/agent_group.hxx"
@@ -28,6 +29,7 @@
 #include "core/cluster.hxx"
 #include "core/impl/subdoc/command.hxx"
 #include "core/logger/logger.hxx"
+#include "core/metrics/meter_wrapper.hxx"
 #include "core/operations/document_append.hxx"
 #include "core/operations/document_decrement.hxx"
 #include "core/operations/document_exists.hxx"
@@ -53,6 +55,8 @@
 #include "core/range_scan_orchestrator.hxx"
 #include "core/range_scan_orchestrator_options.hxx"
 #include "core/topology/configuration.hxx"
+#include "core/tracing/constants.hxx"
+#include "core/tracing/tracer_wrapper.hxx"
 
 #include <couchbase/binary_collection.hxx>
 #include <couchbase/cas.hxx>
@@ -103,6 +107,8 @@
 #include <variant>
 #include <vector>
 
+#include "core/tracing/attribute_helpers.hxx"
+
 namespace couchbase
 {
 class collection_impl : public std::enable_shared_from_this<collection_impl>
@@ -148,41 +154,62 @@ public:
 
   void get(std::string document_key, get_options::built options, get_handler&& handler) const
   {
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_get, options.parent_span);
+
     if (!options.with_expiry && options.projections.empty()) {
-      return core_.execute(
-        core::operations::get_request{
-          core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-          {},
-          {},
-          options.timeout,
-          { options.retry_strategy },
-          options.parent_span,
+      core::operations::get_request request{
+        core::document_id{
+          bucket_name_,
+          scope_name_,
+          name_,
+          std::move(document_key),
         },
-        [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
+        {},
+        {},
+        options.timeout,
+        { options.retry_strategy },
+        obs_rec->operation_span(),
+      };
+      return core_.execute(
+        std::move(request),
+        [obs_rec = std::move(obs_rec),
+         crypto_manager = crypto_manager_,
+         handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           return handler(
             core::impl::make_error(std::move(resp.ctx)),
             get_result{
               resp.cas, { std::move(resp.value), resp.flags }, {}, std::move(crypto_manager) });
         });
     }
-    return core_.execute(
-      core::operations::get_projected_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        options.projections,
-        options.with_expiry,
-        {},
-        false,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    core::operations::get_projected_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
+      {},
+      {},
+      options.projections,
+      options.with_expiry,
+      {},
+      false,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       crypto_manager = crypto_manager_,
+       handler = std::move(handler)](auto resp) mutable {
         std::optional<std::chrono::system_clock::time_point> expiry_time{};
         if (resp.expiry && resp.expiry.value() > 0) {
           expiry_time.emplace(std::chrono::seconds{ resp.expiry.value() });
         }
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)),
                        get_result{ resp.cas,
                                    { std::move(resp.value), resp.flags },
@@ -196,17 +223,30 @@ public:
                      get_and_touch_options::built options,
                      get_and_touch_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::get_and_touch_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        expiry,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_and_touch,
+                                                 options.parent_span);
+
+    core::operations::get_and_touch_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
+      {},
+      {},
+      expiry,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       crypto_manager = crypto_manager_,
+       handler = std::move(handler)](auto resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(
           core::impl::make_error(std::move(resp.ctx)),
           get_result{
@@ -219,17 +259,27 @@ public:
              touch_options::built options,
              touch_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::touch_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        expiry,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_touch, options.parent_span);
+
+    core::operations::touch_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](const auto& resp) mutable {
+      {},
+      {},
+      expiry,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](const auto& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)), result{ resp.cas });
       });
   }
@@ -238,13 +288,26 @@ public:
                        const get_any_replica_options::built& options,
                        core::impl::movable_get_any_replica_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::get_any_replica_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        options.timeout,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_any_replica,
+                                                 options.parent_span);
+
+    core::operations::get_any_replica_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
+      options.timeout,
+      options.read_preference,
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       crypto_manager = crypto_manager_,
+       handler = std::move(handler)](auto resp) mutable {
+        obs_rec->finish(resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)),
                        get_replica_result{
                          resp.cas,
@@ -259,13 +322,25 @@ public:
                         const get_all_replicas_options::built& options,
                         core::impl::movable_get_all_replicas_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::get_all_replicas_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        options.timeout,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_all_replicas,
+                                                 options.parent_span);
+
+    core::operations::get_all_replicas_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto resp) mutable {
+      options.timeout,
+      options.read_preference,
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       crypto_manager = crypto_manager_,
+       handler = std::move(handler)](auto resp) mutable {
         get_all_replicas_result result{};
         for (auto& entry : resp.entries) {
           result.emplace_back(get_replica_result{
@@ -275,6 +350,7 @@ public:
             crypto_manager,
           });
         }
+        obs_rec->finish(resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)), std::move(result));
       });
   }
@@ -283,6 +359,9 @@ public:
               remove_options::built options,
               remove_handler&& handler) const
   {
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_remove, options.parent_span);
+
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -290,18 +369,20 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::remove_request request{
+        std::move(id),
+        {},
+        {},
+        options.cas,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::remove_request{
-          std::move(id),
-          {},
-          {},
-          options.cas,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
             return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
           }
@@ -318,33 +399,39 @@ public:
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
-      options.parent_span,
+      obs_rec->operation_span(),
     };
-    return core_.execute(std::move(request),
-                         [core = core_, id = std::move(id), options, handler = std::move(handler)](
-                           auto&& resp) mutable {
-                           if (resp.ctx.ec()) {
-                             return handler(core::impl::make_error(std::move(resp.ctx)),
-                                            mutation_result{ resp.cas, std::move(resp.token) });
-                           }
-                           auto token = resp.token;
-                           core::impl::initiate_observe_poll(
-                             core,
-                             std::move(id),
-                             token,
-                             options.timeout,
-                             options.persist_to,
-                             options.replicate_to,
-                             [resp, handler = std::move(handler)](std::error_code ec) mutable {
-                               if (ec) {
-                                 resp.ctx.override_ec(ec);
-                                 return handler(core::impl::make_error(std::move(resp.ctx)),
-                                                mutation_result{});
-                               }
-                               return handler(core::impl::make_error(std::move(resp.ctx)),
-                                              mutation_result{ resp.cas, std::move(resp.token) });
-                             });
-                         });
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto&& resp) mutable {
+        if (resp.ctx.ec()) {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+          return handler(core::impl::make_error(std::move(resp.ctx)),
+                         mutation_result{ resp.cas, std::move(resp.token) });
+        }
+        auto token = resp.token;
+        core::impl::initiate_observe_poll(
+          core,
+          std::move(id),
+          token,
+          options.timeout,
+          options.persist_to,
+          options.replicate_to,
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
+            if (ec) {
+              resp.ctx.override_ec(ec);
+              return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
+            }
+            return handler(core::impl::make_error(std::move(resp.ctx)),
+                           mutation_result{ resp.cas, std::move(resp.token) });
+          });
+      });
   }
 
   void get_and_lock(std::string document_key,
@@ -352,17 +439,29 @@ public:
                     get_and_lock_options::built options,
                     get_and_lock_handler&& handler) const
   {
-    core_.execute(
-      core::operations::get_and_lock_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        static_cast<uint32_t>(lock_duration.count()),
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec = create_observability_recorder(core::tracing::operation::mcbp_get_and_lock,
+                                                 options.parent_span);
+
+    core::operations::get_and_lock_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [crypto_manager = crypto_manager_, handler = std::move(handler)](auto&& resp) mutable {
+      {},
+      {},
+      static_cast<uint32_t>(lock_duration.count()),
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec),
+       crypto_manager = crypto_manager_,
+       handler = std::move(handler)](auto&& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(
           core::impl::make_error(std::move(resp.ctx)),
           get_result{
@@ -375,17 +474,27 @@ public:
               unlock_options::built options,
               unlock_handler&& handler) const
   {
-    core_.execute(
-      core::operations::unlock_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        cas,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_unlock, options.parent_span);
+
+    core::operations::unlock_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto&& resp) mutable {
+      {},
+      {},
+      cas,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto&& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)));
       });
   }
@@ -394,16 +503,26 @@ public:
               exists_options::built options,
               exists_handler&& handler) const
   {
-    core_.execute(
-      core::operations::exists_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        {},
-        {},
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_exists, options.parent_span);
+
+    core::operations::exists_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto&& resp) mutable {
+      {},
+      {},
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto&& resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)),
                        exists_result{ resp.cas, resp.exists() });
       });
@@ -414,23 +533,29 @@ public:
                  lookup_in_options::built options,
                  lookup_in_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::lookup_in_request{
-        core::document_id{
-          bucket_name_,
-          scope_name_,
-          name_,
-          std::move(document_key),
-        },
-        {},
-        {},
-        options.access_deleted,
-        specs,
-        options.timeout,
-        { options.retry_strategy },
-        options.parent_span,
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_lookup_in, options.parent_span);
+
+    core::operations::lookup_in_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto resp) mutable {
+      {},
+      {},
+      options.access_deleted,
+      specs,
+      options.timeout,
+      { options.retry_strategy },
+      obs_rec->operation_span(),
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+        obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
+
         if (resp.ctx.ec()) {
           return handler(core::impl::make_error(std::move(resp.ctx)), lookup_in_result{});
         }
@@ -456,15 +581,24 @@ public:
                               const lookup_in_all_replicas_options::built& options,
                               lookup_in_all_replicas_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::lookup_in_all_replicas_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        specs,
-        options.timeout,
-        options.parent_span,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_lookup_in_all_replicas, options.parent_span);
+
+    core::operations::lookup_in_all_replicas_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto resp) mutable {
+      specs,
+      options.timeout,
+      obs_rec->operation_span(),
+      options.read_preference,
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
         lookup_in_all_replicas_result result{};
         for (auto& res : resp.entries) {
           std::vector<lookup_in_result::entry> entries;
@@ -485,6 +619,7 @@ public:
             res.is_replica,
           });
         }
+        obs_rec->finish(resp.ctx.ec());
         return handler(core::impl::make_error(std::move(resp.ctx)), result);
       });
   }
@@ -494,15 +629,24 @@ public:
                              const lookup_in_any_replica_options::built& options,
                              lookup_in_any_replica_handler&& handler) const
   {
-    return core_.execute(
-      core::operations::lookup_in_any_replica_request{
-        core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) },
-        specs,
-        options.timeout,
-        options.parent_span,
-        options.read_preference,
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_lookup_in_any_replica, options.parent_span);
+
+    core::operations::lookup_in_any_replica_request request{
+      core::document_id{
+        bucket_name_,
+        scope_name_,
+        name_,
+        std::move(document_key),
       },
-      [handler = std::move(handler)](auto resp) mutable {
+      specs,
+      options.timeout,
+      obs_rec->operation_span(),
+      options.read_preference,
+    };
+    return core_.execute(
+      std::move(request),
+      [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
         std::vector<lookup_in_result::entry> entries;
         entries.reserve(resp.fields.size());
         for (auto& field : resp.fields) {
@@ -514,7 +658,7 @@ public:
             field.ec,
           });
         }
-        entries.reserve(resp.fields.size());
+        obs_rec->finish(resp.ctx.ec());
         return handler(
           core::impl::make_error(std::move(resp.ctx)),
           lookup_in_replica_result{ resp.cas, std::move(entries), resp.deleted, resp.is_replica });
@@ -526,6 +670,9 @@ public:
                  mutate_in_options::built options,
                  mutate_in_handler&& handler) const
   {
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_mutate_in, options.parent_span, options.durability_level);
+
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -533,25 +680,27 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::mutate_in_request request{
+        std::move(id),
+        {},
+        {},
+        options.cas,
+        options.access_deleted,
+        options.create_as_deleted,
+        false,
+        options.expiry,
+        options.store_semantics,
+        specs,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        options.preserve_expiry,
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::mutate_in_request{
-          std::move(id),
-          {},
-          {},
-          options.cas,
-          options.access_deleted,
-          options.create_as_deleted,
-          false,
-          options.expiry,
-          options.store_semantics,
-          specs,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.preserve_expiry,
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
             return handler(core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
           }
@@ -585,13 +734,17 @@ public:
       options.timeout,
       { options.retry_strategy },
       options.preserve_expiry,
-      options.parent_span,
+      obs_rec->operation_span(),
     };
     return core_.execute(
       std::move(request),
-      [core = core_, id = std::move(id), options, handler = std::move(handler)](
-        auto&& resp) mutable {
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto&& resp) mutable {
         if (resp.ctx.ec()) {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           return handler(core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
         }
 
@@ -603,7 +756,9 @@ public:
           options.timeout,
           options.persist_to,
           options.replicate_to,
-          [resp, handler = std::move(handler)](std::error_code ec) mutable {
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
             if (ec) {
               resp.ctx.override_ec(ec);
               return handler(core::impl::make_error(std::move(resp.ctx)), mutate_in_result{});
@@ -625,11 +780,14 @@ public:
   }
 
   void upsert(std::string document_key,
-              codec::encoded_value encoded,
+              std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
               upsert_options::built options,
               upsert_handler&& handler) const
   {
-    auto value = std::move(encoded);
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_upsert, options.parent_span, options.durability_level);
+
+    auto [data, flags] = get_encoded_value(std::move(value), obs_rec);
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -637,21 +795,23 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::upsert_request request{
+        std::move(id),
+        std::move(data),
+        {},
+        {},
+        flags,
+        options.expiry,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        options.preserve_expiry,
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::upsert_request{
-          std::move(id),
-          std::move(value.data),
-          {},
-          {},
-          value.flags,
-          options.expiry,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.preserve_expiry,
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           return handler(core::impl::make_error(std::move(resp.ctx)),
                          mutation_result{ resp.cas, std::move(resp.token) });
         });
@@ -659,21 +819,26 @@ public:
 
     core::operations::upsert_request request{
       id,
-      std::move(value.data),
+      std::move(data),
       {},
       {},
-      value.flags,
+      flags,
       options.expiry,
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
       options.preserve_expiry,
-      options.parent_span,
+      obs_rec->operation_span(),
     };
     return core_.execute(
       std::move(request),
-      [core = core_, id = std::move(id), options, handler = std::move(handler)](auto resp) mutable {
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto resp) mutable {
         if (resp.ctx.ec()) {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           return handler(core::impl::make_error(std::move(resp.ctx)),
                          mutation_result{ resp.cas, std::move(resp.token) });
         }
@@ -686,7 +851,9 @@ public:
           options.timeout,
           options.persist_to,
           options.replicate_to,
-          [resp, handler = std::move(handler)](std::error_code ec) mutable {
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
             if (ec) {
               resp.ctx.override_ec(ec);
               return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
@@ -698,11 +865,14 @@ public:
   }
 
   void insert(std::string document_key,
-              codec::encoded_value encoded,
+              std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
               insert_options::built options,
               insert_handler&& handler) const
   {
-    auto value = std::move(encoded);
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_insert, options.parent_span, options.durability_level);
+
+    auto [data, flags] = get_encoded_value(std::move(value), obs_rec);
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -710,20 +880,22 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::insert_request request{
+        std::move(id),
+        std::move(data),
+        {},
+        {},
+        flags,
+        options.expiry,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::insert_request{
-          std::move(id),
-          std::move(value.data),
-          {},
-          {},
-          value.flags,
-          options.expiry,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto&& resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto&& resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
             return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
           }
@@ -734,20 +906,25 @@ public:
 
     core::operations::insert_request request{
       id,
-      std::move(value.data),
+      std::move(data),
       {},
       {},
-      value.flags,
+      flags,
       options.expiry,
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
-      options.parent_span,
+      obs_rec->operation_span(),
     };
     return core_.execute(
       std::move(request),
-      [core = core_, id = std::move(id), options, handler = std::move(handler)](auto resp) mutable {
+      [obs_rec = std::move(obs_rec),
+       core = core_,
+       id = std::move(id),
+       options,
+       handler = std::move(handler)](auto resp) mutable {
         if (resp.ctx.ec()) {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           return handler(core::impl::make_error(std::move(resp.ctx)),
                          mutation_result{ resp.cas, std::move(resp.token) });
         }
@@ -760,7 +937,9 @@ public:
           options.timeout,
           options.persist_to,
           options.replicate_to,
-          [resp, handler = std::move(handler)](std::error_code ec) mutable {
+          [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+            std::error_code ec) mutable {
+            obs_rec->finish(resp.ctx.retry_attempts(), ec);
             if (ec) {
               resp.ctx.override_ec(ec);
               return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
@@ -770,12 +949,17 @@ public:
           });
       });
   }
+
   void replace(std::string document_key,
-               codec::encoded_value encoded,
+               std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
                replace_options::built options,
                replace_handler&& handler) const
   {
-    auto value = std::move(encoded);
+    auto obs_rec = create_observability_recorder(
+      core::tracing::operation::mcbp_replace, options.parent_span, options.durability_level);
+
+    auto [data, flags] = get_encoded_value(std::move(value), obs_rec);
+
     auto id = core::document_id{
       bucket_name_,
       scope_name_,
@@ -783,22 +967,24 @@ public:
       std::move(document_key),
     };
     if (options.persist_to == persist_to::none && options.replicate_to == replicate_to::none) {
+      core::operations::replace_request request{
+        std::move(id),
+        std::move(data),
+        {},
+        {},
+        flags,
+        options.expiry,
+        options.cas,
+        options.durability_level,
+        options.timeout,
+        { options.retry_strategy },
+        options.preserve_expiry,
+        obs_rec->operation_span(),
+      };
       return core_.execute(
-        core::operations::replace_request{
-          std::move(id),
-          std::move(value.data),
-          {},
-          {},
-          value.flags,
-          options.expiry,
-          options.cas,
-          options.durability_level,
-          options.timeout,
-          { options.retry_strategy },
-          options.preserve_expiry,
-          options.parent_span,
-        },
-        [handler = std::move(handler)](auto resp) mutable {
+        std::move(request),
+        [obs_rec = std::move(obs_rec), handler = std::move(handler)](auto resp) mutable {
+          obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
           if (resp.ctx.ec()) {
             return handler(core::impl::make_error(std::move(resp.ctx)), mutation_result{});
           }
@@ -809,22 +995,26 @@ public:
 
     core::operations::replace_request request{
       id,
-      std::move(value.data),
+      std::move(data),
       {},
       {},
-      value.flags,
+      flags,
       options.expiry,
       options.cas,
       durability_level::none,
       options.timeout,
       { options.retry_strategy },
       options.preserve_expiry,
-      options.parent_span,
+      obs_rec->operation_span(),
     };
     return core_.execute(std::move(request),
-                         [core = core_, id = std::move(id), options, handler = std::move(handler)](
-                           auto&& resp) mutable {
+                         [obs_rec = std::move(obs_rec),
+                          core = core_,
+                          id = std::move(id),
+                          options,
+                          handler = std::move(handler)](auto&& resp) mutable {
                            if (resp.ctx.ec()) {
+                             obs_rec->finish(resp.ctx.retry_attempts(), resp.ctx.ec());
                              return handler(core::impl::make_error(std::move(resp.ctx)),
                                             mutation_result{ resp.cas, std::move(resp.token) });
                            }
@@ -837,7 +1027,9 @@ public:
                              options.timeout,
                              options.persist_to,
                              options.replicate_to,
-                             [resp, handler = std::move(handler)](std::error_code ec) mutable {
+                             [obs_rec = std::move(obs_rec), resp, handler = std::move(handler)](
+                               std::error_code ec) mutable {
+                               obs_rec->finish(resp.ctx.retry_attempts(), ec);
                                if (ec) {
                                  resp.ctx.override_ec(ec);
                                  return handler(core::impl::make_error(std::move(resp.ctx)),
@@ -851,6 +1043,9 @@ public:
 
   void scan(scan_type::built scan_type, scan_options::built options, scan_handler&& handler) const
   {
+    auto obs_rec =
+      create_observability_recorder(core::tracing::operation::mcbp_scan, options.parent_span);
+
     core::range_scan_orchestrator_options orchestrator_opts{ options.ids_only };
     if (!options.mutation_state.empty()) {
       orchestrator_opts.consistent_with = core::mutation_state{ options.mutation_state };
@@ -903,22 +1098,31 @@ public:
 
     return core_.open_bucket(
       bucket_name_,
-      [this, handler = std::move(handler), orchestrator_opts, core_scan_type](
-        std::error_code ec) mutable {
+      [this,
+       obs_rec = std::move(obs_rec),
+       handler = std::move(handler),
+       orchestrator_opts,
+       core_scan_type](std::error_code ec) mutable {
         if (ec) {
+          obs_rec->finish(ec);
           return handler(error(ec), {});
         }
         return core_.with_bucket_configuration(
           bucket_name_,
-          [this, handler = std::move(handler), orchestrator_opts, core_scan_type](
-            std::error_code ec,
-            const std::shared_ptr<core::topology::configuration>& config) mutable {
+          [this,
+           obs_rec = std::move(obs_rec),
+           handler = std::move(handler),
+           orchestrator_opts,
+           core_scan_type](std::error_code ec,
+                           const std::shared_ptr<core::topology::configuration>& config) mutable {
             if (ec) {
+              obs_rec->finish(ec);
               return handler(
                 error(ec, "An error occurred when attempting to fetch the bucket configuration."),
                 {});
             }
             if (!config->capabilities.supports_range_scan()) {
+              obs_rec->finish(ec);
               return handler(error(errc::common::feature_not_available,
                                    "This bucket does not support range scan."),
                              {});
@@ -927,6 +1131,7 @@ public:
               core::agent_group(core_.io_context(), core::agent_group_config{ { core_ } });
             ec = agent_group.open_bucket(bucket_name_);
             if (ec) {
+              obs_rec->finish(ec);
               return handler(error(ec,
                                    fmt::format("An error occurred while opening the `{}` bucket.",
                                                bucket_name_)),
@@ -934,6 +1139,7 @@ public:
             }
             auto agent = agent_group.get_agent(bucket_name_);
             if (!agent.has_value()) {
+              obs_rec->finish(ec);
               return handler(
                 error(agent.error(),
                       fmt::format(
@@ -944,6 +1150,7 @@ public:
             if (!config->vbmap.has_value() || config->vbmap->empty()) {
               CB_LOG_WARNING("Unable to get vbucket map for `{}` - cannot perform scan operation",
                              bucket_name_);
+              obs_rec->finish(ec);
               return handler(error(errc::common::request_canceled,
                                    "No vbucket map included with the bucket config"),
                              {});
@@ -957,8 +1164,10 @@ public:
                                                               core_scan_type,
                                                               orchestrator_opts);
             return orchestrator.scan(
-              [crypto_manager = crypto_manager_,
+              [obs_rec = std::move(obs_rec),
+               crypto_manager = crypto_manager_,
                handler = std::move(handler)](auto ec, auto core_scan_result) mutable {
+                obs_rec->finish(ec);
                 if (ec) {
                   return handler(error(ec, "Error while starting the range scan"), {});
                 }
@@ -971,6 +1180,39 @@ public:
   }
 
 private:
+  static auto get_encoded_value(
+    std::variant<codec::encoded_value, std::function<codec::encoded_value()>> value,
+    const std::unique_ptr<core::impl::observability_recorder>& obs_rec) -> codec::encoded_value
+  {
+    if (std::holds_alternative<codec::encoded_value>(value)) {
+      return std::get<codec::encoded_value>(value);
+    }
+    const auto request_encoding_span = obs_rec->create_request_encoding_span();
+    auto encoded = std::get<std::function<codec::encoded_value()>>(value)();
+    request_encoding_span->end();
+    return encoded;
+  }
+
+  [[nodiscard]] auto create_observability_recorder(
+    const std::string& operation_name,
+    const std::shared_ptr<tracing::request_span>& parent_span,
+    const std::optional<durability_level> durability = {}) const
+    -> std::unique_ptr<core::impl::observability_recorder>
+  {
+    auto rec = core::impl::observability_recorder::create(
+      operation_name, parent_span, core_.tracer(), core_.meter());
+
+    rec->with_service(core::tracing::service::key_value);
+    rec->with_bucket_name(bucket_name_);
+    rec->with_scope_name(scope_name_);
+    rec->with_collection_name(name_);
+    if (durability.has_value()) {
+      rec->with_durability(durability.value());
+    }
+
+    return rec;
+  }
+
   core::cluster core_;
   std::string bucket_name_;
   std::string scope_name_;
@@ -1392,6 +1634,30 @@ collection::upsert(std::string document_id,
   return future;
 }
 
+auto
+collection::upsert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const upsert_options& options,
+                   upsert_handler&& handler) const -> void
+{
+  impl_->upsert(
+    std::move(document_id), std::move(document_fn), options.build(), std::move(handler));
+}
+
+auto
+collection::upsert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const upsert_options& options) const
+  -> std::future<std::pair<error, mutation_result>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, mutation_result>>>();
+  auto future = barrier->get_future();
+  upsert(std::move(document_id), std::move(document_fn), options, [barrier](auto err, auto result) {
+    barrier->set_value({ std::move(err), std::move(result) });
+  });
+  return future;
+}
+
 void
 collection::insert(std::string document_id,
                    codec::encoded_value document,
@@ -1416,6 +1682,30 @@ collection::insert(std::string document_id,
   return future;
 }
 
+auto
+collection::insert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const insert_options& options,
+                   insert_handler&& handler) const -> void
+{
+  impl_->insert(
+    std::move(document_id), std::move(document_fn), options.build(), std::move(handler));
+}
+
+auto
+collection::insert(std::string document_id,
+                   std::function<codec::encoded_value()> document_fn,
+                   const insert_options& options) const
+  -> std::future<std::pair<error, mutation_result>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, mutation_result>>>();
+  auto future = barrier->get_future();
+  insert(std::move(document_id), std::move(document_fn), options, [barrier](auto err, auto result) {
+    barrier->set_value({ std::move(err), std::move(result) });
+  });
+  return future;
+}
+
 void
 collection::replace(std::string document_id,
                     codec::encoded_value document,
@@ -1440,6 +1730,31 @@ collection::replace(std::string document_id,
   return future;
 }
 
+void
+collection::replace(std::string document_id,
+                    std::function<codec::encoded_value()> document_fn,
+                    const replace_options& options,
+                    replace_handler&& handler) const
+{
+  impl_->replace(
+    std::move(document_id), std::move(document_fn), options.build(), std::move(handler));
+}
+
+auto
+collection::replace(std::string document_id,
+                    std::function<codec::encoded_value()> document_fn,
+                    const replace_options& options) const
+  -> std::future<std::pair<error, mutation_result>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, mutation_result>>>();
+  auto future = barrier->get_future();
+  replace(
+    std::move(document_id), std::move(document_fn), options, [barrier](auto err, auto result) {
+      barrier->set_value({ std::move(err), std::move(result) });
+    });
+  return future;
+}
+
 void
 collection::scan(const couchbase::scan_type& scan_type,
                  const couchbase::scan_options& options,