couchbase 3.7.0 → 3.8.0

data/ext/couchbase/core/tracing/threshold_logging_tracer.cxx

@@ -24,14 +24,14 @@
 #include "core/meta/version.hxx"
 #include "core/platform/uuid.h"
 #include "core/service_type_fmt.hxx"
+#include "core/utils/concurrent_fixed_priority_queue.hxx"
 #include "core/utils/json.hxx"
 
 #include <asio/steady_timer.hpp>
+#include <memory>
 #include <tao/json/value.hpp>
 
 #include <chrono>
-#include <mutex>
-#include <queue>
 #include <utility>
 
 namespace couchbase::core::tracing
@@ -44,6 +44,11 @@ struct reported_span {
   {
     return duration < other.duration;
   }
+
+  auto operator>(const reported_span& other) const -> bool
+  {
+    return duration > other.duration;
+  }
 };
 
 class threshold_logging_span
@@ -52,16 +57,15 @@ class threshold_logging_span
 {
 private:
   std::chrono::system_clock::time_point start_{ std::chrono::system_clock::now() };
-  std::string id_{ uuid::to_string(uuid::random()) };
-  std::map<std::string, std::uint64_t> integer_tags_{};
-  std::map<std::string, std::string> string_tags_{
-    { attributes::system, "couchbase" },
-    { attributes::span_kind, "client" },
-    { attributes::component, couchbase::core::meta::sdk_id() },
-  };
-  std::chrono::microseconds duration_{ 0 };
+  std::chrono::microseconds total_duration_{ 0 };
+
   std::uint64_t last_server_duration_us_{ 0 };
   std::uint64_t total_server_duration_us_{ 0 };
+  std::optional<std::string> operation_id_{};
+  std::optional<std::string> last_local_id_{};
+  std::optional<std::string> service_{};
+  std::optional<std::string> peer_hostname_{};
+  std::optional<std::uint16_t> peer_port_{};
 
   std::shared_ptr<threshold_logging_tracer> tracer_{};
 
@@ -74,30 +78,74 @@ public:
   {
   }
 
-  void add_tag(const std::string& name, std::uint64_t value) override
+  void add_tag(const std::string& tag_name, std::uint64_t value) override
   {
-    if (name == tracing::attributes::server_duration) {
+    if (tag_name == tracing::attributes::dispatch::server_duration) {
       last_server_duration_us_ = value;
-      total_server_duration_us_ += value;
+      if (name() != tracing::operation::step_dispatch) {
+        total_server_duration_us_ += value;
+      }
+    }
+    if (tag_name == tracing::attributes::dispatch::peer_port) {
+      peer_port_ = static_cast<std::uint16_t>(value);
     }
-    integer_tags_.try_emplace(name, value);
   }
 
-  void add_tag(const std::string& name, const std::string& value) override
+  void add_tag(const std::string& tag_name, const std::string& value) override
   {
-    string_tags_.try_emplace(name, value);
+    if (tag_name == tracing::attributes::op::service) {
+      service_ = value;
+    }
+    if (tag_name == tracing::attributes::dispatch::local_id) {
+      last_local_id_ = value;
+    }
+    if (tag_name == tracing::attributes::dispatch::operation_id) {
+      operation_id_ = value;
+    }
+    if (tag_name == tracing::attributes::dispatch::peer_address) {
+      peer_hostname_ = value;
+    }
   }
 
   void end() override;
 
-  [[nodiscard]] auto string_tags() const -> const auto&
+  void set_last_local_id(const std::string& id)
+  {
+    last_local_id_ = id;
+  }
+
+  void set_operation_id(const std::string& id)
+  {
+    operation_id_ = id;
+  }
+
+  void set_peer_hostname(const std::string& hostname)
+  {
+    peer_hostname_ = hostname;
+  }
+
+  void set_peer_port(const std::uint16_t port)
+  {
+    peer_port_ = port;
+  }
+
+  [[nodiscard]] auto last_remote_socket() const -> std::optional<std::string>
+  {
+    if (peer_hostname_.has_value() && peer_port_.has_value()) {
+      return fmt::format("{}:{}", peer_hostname_.value(), peer_port_.value());
+    }
+    return {};
+  }
+
+  void add_server_duration(const std::uint64_t duration_us)
   {
-    return string_tags_;
+    last_server_duration_us_ = duration_us;
+    total_server_duration_us_ += duration_us;
   }
 
-  [[nodiscard]] auto duration() const -> std::chrono::microseconds
+  [[nodiscard]] auto total_duration() const -> std::chrono::microseconds
   {
-    return duration_;
+    return total_duration_;
   }
 
   [[nodiscard]] auto last_server_duration_us() const -> std::uint64_t
@@ -110,27 +158,27 @@ public:
     return total_server_duration_us_;
   }
 
-  [[nodiscard]] auto orphan() const -> bool
+  [[nodiscard]] auto operation_id() const -> std::optional<std::string>
   {
-    return string_tags_.find(tracing::attributes::orphan) != string_tags_.end();
+    return operation_id_;
+  }
+
+  [[nodiscard]] auto last_local_id() const -> std::optional<std::string>
+  {
+    return last_local_id_;
   }
 
   [[nodiscard]] auto is_key_value() const -> bool
   {
-    auto service_tag = string_tags_.find(tracing::attributes::service);
-    if (service_tag == string_tags_.end()) {
-      return false;
-    }
-    return service_tag->second == tracing::service::key_value;
+    return service_.has_value() && service_.value() == tracing::service::key_value;
   }
 
   [[nodiscard]] auto service() const -> std::optional<service_type>
   {
-    auto service_tag = string_tags_.find(tracing::attributes::service);
-    if (service_tag == string_tags_.end()) {
+    if (!service_.has_value()) {
       return {};
     }
-    const auto& service_name = service_tag->second;
+    const auto& service_name = service_.value();
     if (service_name == tracing::service::key_value) {
       return service_type::key_value;
     }
@@ -153,65 +201,7 @@ public:
   }
 };
 
-template<typename T>
-class concurrent_fixed_queue
-{
-private:
-  std::mutex mutex_;
-  std::priority_queue<T> data_;
-  std::size_t capacity_{};
-
-public:
-  using size_type = typename std::priority_queue<T>::size_type;
-
-  explicit concurrent_fixed_queue(std::size_t capacity)
-    : capacity_(capacity)
-  {
-  }
-
-  concurrent_fixed_queue(const concurrent_fixed_queue&) = delete;
-  concurrent_fixed_queue(concurrent_fixed_queue&&) = delete;
-  auto operator=(const concurrent_fixed_queue&) -> concurrent_fixed_queue& = delete;
-  auto operator=(concurrent_fixed_queue&&) -> concurrent_fixed_queue& = delete;
-  ~concurrent_fixed_queue() = default;
-
-  void pop()
-  {
-    std::unique_lock<std::mutex> lock(mutex_);
-    data_.pop();
-  }
-
-  auto size() -> size_type
-  {
-    std::unique_lock<std::mutex> lock(mutex_);
-    return data_.size();
-  }
-
-  auto empty() -> bool
-  {
-    const std::unique_lock<std::mutex> lock(mutex_);
-    return data_.empty();
-  }
-
-  void emplace(const T&& item)
-  {
-    const std::unique_lock<std::mutex> lock(mutex_);
-    data_.emplace(std::forward<const T>(item));
-    if (data_.size() > capacity_) {
-      data_.pop();
-    }
-  }
-
-  auto steal_data() -> std::priority_queue<T>
-  {
-    std::priority_queue<T> data;
-    const std::unique_lock<std::mutex> lock(mutex_);
-    std::swap(data, data_);
-    return data;
-  }
-};
-
-using fixed_span_queue = concurrent_fixed_queue<reported_span>;
+using fixed_span_queue = utils::concurrent_fixed_priority_queue<reported_span>;
 
 auto
 convert(const std::shared_ptr<threshold_logging_span>& span) -> reported_span
@@ -219,45 +209,35 @@ convert(const std::shared_ptr<threshold_logging_span>& span) -> reported_span
   tao::json::value entry{
     { "operation_name", span->name() },
     { "total_duration_us",
-      std::chrono::duration_cast<std::chrono::microseconds>(span->duration()).count() }
+      std::chrono::duration_cast<std::chrono::microseconds>(span->total_duration()).count() }
   };
   if (span->is_key_value()) {
     entry["last_server_duration_us"] = span->last_server_duration_us();
     entry["total_server_duration_us"] = span->total_server_duration_us();
   }
 
-  const auto& tags = span->string_tags();
-  auto pair = tags.find(attributes::operation_id);
-  if (pair != tags.end()) {
-    entry["last_operation_id"] = pair->second;
+  if (span->operation_id().has_value()) {
+    entry["last_operation_id"] = span->operation_id().value();
   }
 
-  pair = tags.find(attributes::local_id);
-  if (pair != tags.end()) {
-    entry["last_local_id"] = pair->second;
+  if (span->last_local_id().has_value()) {
+    entry["last_local_id"] = span->last_local_id().value();
   }
 
-  pair = tags.find(attributes::local_socket);
-  if (pair != tags.end()) {
-    entry["last_local_socket"] = pair->second;
+  if (span->last_remote_socket().has_value()) {
+    entry["last_remote_socket"] = span->last_remote_socket().value();
   }
 
-  pair = tags.find(attributes::remote_socket);
-  if (pair != tags.end()) {
-    entry["last_remote_socket"] = pair->second;
-  }
-
-  return { span->duration(), std::move(entry) };
+  return { span->total_duration(), std::move(entry) };
 }
 
 class threshold_logging_tracer_impl
+  : public std::enable_shared_from_this<threshold_logging_tracer_impl>
 {
 public:
   threshold_logging_tracer_impl(const threshold_logging_options& options, asio::io_context& ctx)
     : options_(options)
-    , emit_orphan_report_(ctx)
     , emit_threshold_report_(ctx)
-    , orphan_queue_{ options.orphaned_sample_size }
   {
     threshold_queues_.try_emplace(service_type::key_value, options.threshold_sample_size);
     threshold_queues_.try_emplace(service_type::query, options.threshold_sample_size);
@@ -274,98 +254,55 @@ public:
 
   ~threshold_logging_tracer_impl()
   {
-    emit_orphan_report_.cancel();
     emit_threshold_report_.cancel();
 
-    log_orphan_report();
     log_threshold_report();
   }
 
   void start()
   {
-    rearm_orphan_reporter();
     rearm_threshold_reporter();
   }
 
   void stop()
   {
-    emit_orphan_report_.cancel();
     emit_threshold_report_.cancel();
   }
 
-  void add_orphan(const std::shared_ptr<threshold_logging_span>& span)
-  {
-    orphan_queue_.emplace(convert(span));
-  }
-
   void check_threshold(const std::shared_ptr<threshold_logging_span>& span)
   {
-    auto service = span->service();
+    const auto service = span->service();
     if (!service.has_value()) {
       return;
     }
-    if (span->duration() > options_.threshold_for_service(service.value())) {
-      auto queue = threshold_queues_.find(service.value());
-      if (queue != threshold_queues_.end()) {
+    if (span->total_duration() > options_.threshold_for_service(service.value())) {
+      if (const auto queue = threshold_queues_.find(service.value());
+          queue != threshold_queues_.end()) {
         queue->second.emplace(convert(span));
       }
     }
   }
 
 private:
-  void rearm_orphan_reporter()
-  {
-    emit_orphan_report_.expires_after(options_.orphaned_emit_interval);
-    emit_orphan_report_.async_wait([this](std::error_code ec) {
-      if (ec == asio::error::operation_aborted) {
-        return;
-      }
-      log_orphan_report();
-      rearm_orphan_reporter();
-    });
-  }
-
   void rearm_threshold_reporter()
   {
     emit_threshold_report_.expires_after(options_.threshold_emit_interval);
-    emit_threshold_report_.async_wait([this](std::error_code ec) {
+    emit_threshold_report_.async_wait([self = shared_from_this()](std::error_code ec) -> void {
       if (ec == asio::error::operation_aborted) {
         return;
       }
-      log_threshold_report();
-      rearm_threshold_reporter();
+      self->log_threshold_report();
+      self->rearm_threshold_reporter();
     });
   }
 
-  void log_orphan_report()
-  {
-    if (orphan_queue_.empty()) {
-      return;
-    }
-    auto queue = orphan_queue_.steal_data();
-    tao::json::value report{
-      { "count", queue.size() },
-#if COUCHBASE_CXX_CLIENT_DEBUG_BUILD
-      { "emit_interval_ms", options_.orphaned_emit_interval.count() },
-      { "sample_size", options_.orphaned_sample_size },
-#endif
-    };
-    tao::json::value entries = tao::json::empty_array;
-    while (!queue.empty()) {
-      entries.emplace_back(queue.top().payload);
-      queue.pop();
-    }
-    report["top"] = entries;
-    CB_LOG_WARNING("Orphan responses observed: {}", utils::json::generate(report));
-  }
-
   void log_threshold_report()
   {
     for (auto& [service, threshold_queue] : threshold_queues_) {
       if (threshold_queue.empty()) {
         continue;
       }
-      auto queue = threshold_queue.steal_data();
+      auto [queue, _] = threshold_queue.steal_data();
       tao::json::value report{
         { "count", queue.size() },
         { "service", fmt::format("{}", service) },
@@ -388,11 +325,9 @@ private:
     }
   }
 
-  const threshold_logging_options& options_;
+  threshold_logging_options options_;
 
-  asio::steady_timer emit_orphan_report_;
   asio::steady_timer emit_threshold_report_;
-  fixed_span_queue orphan_queue_;
   std::map<service_type, fixed_span_queue> threshold_queues_{};
 };
 
@@ -407,11 +342,7 @@ threshold_logging_tracer::start_span(std::string name,
 void
 threshold_logging_tracer::report(const std::shared_ptr<threshold_logging_span>& span)
 {
-  if (span->orphan()) {
-    impl_->add_orphan(span);
-  } else {
-    impl_->check_threshold(span);
-  }
+  impl_->check_threshold(span);
 }
 
 threshold_logging_tracer::threshold_logging_tracer(asio::io_context& ctx,
@@ -436,9 +367,30 @@ threshold_logging_tracer::stop()
 void
 threshold_logging_span::end()
 {
-  duration_ = std::chrono::duration_cast<std::chrono::microseconds>(
+  total_duration_ = std::chrono::duration_cast<std::chrono::microseconds>(
     std::chrono::system_clock::now() - start_);
-  tracer_->report(shared_from_this());
+  if (service_.has_value()) {
+    tracer_->report(shared_from_this());
+  }
+  if (name() == tracing::operation::step_dispatch) {
+    // Transfer the relevant attributes to the operation-level span
+    if (const auto p = std::dynamic_pointer_cast<threshold_logging_span>(parent()); p) {
+      if (last_local_id_.has_value()) {
+        p->set_last_local_id(last_local_id_.value());
+      }
+      if (operation_id_.has_value()) {
+        p->set_operation_id(operation_id_.value());
+      }
+      if (peer_hostname_.has_value()) {
+        p->set_peer_hostname(peer_hostname_.value());
+      }
+      if (peer_port_.has_value()) {
+        p->set_peer_port(peer_port_.value());
+      }
+      if (last_server_duration_us_ > 0) {
+        p->add_server_duration(last_server_duration_us_);
+      }
+    }
+  }
 }
-
 } // namespace couchbase::core::tracing

data/ext/couchbase/core/tracing/tracer_wrapper.cxx

@@ -18,13 +18,14 @@
 #include "tracer_wrapper.hxx"
 
 #include "constants.hxx"
-
-#include <mutex>
+#include "core/logger/logger.hxx"
 
 namespace couchbase::core::tracing
 {
-tracer_wrapper::tracer_wrapper(std::shared_ptr<couchbase::tracing::request_tracer> tracer)
+tracer_wrapper::tracer_wrapper(std::shared_ptr<couchbase::tracing::request_tracer> tracer,
+                               std::shared_ptr<cluster_label_listener> label_listener)
   : tracer_{ std::move(tracer) }
+  , cluster_label_listener_{ std::move(label_listener) }
 {
 }
 
@@ -46,42 +47,34 @@ tracer_wrapper::create_span(std::string span_name,
   -> std::shared_ptr<couchbase::tracing::request_span>
 {
   auto span = tracer_->start_span(std::move(span_name), std::move(parent_span));
-  span->add_tag(attributes::system, "couchbase");
-
-  std::optional<std::string> cluster_name;
-  std::optional<std::string> cluster_uuid;
-  {
-    const std::shared_lock lock{ cluster_labels_mutex_ };
-    cluster_name = cluster_name_;
-    cluster_uuid = cluster_uuid_;
+  if (!span->uses_tags()) {
+    return span;
   }
 
+  span->add_tag(attributes::common::system, "couchbase");
+
+  auto [cluster_name, cluster_uuid] = cluster_label_listener_->cluster_labels();
   if (cluster_name) {
-    span->add_tag(attributes::cluster_name, cluster_name.value());
+    span->add_tag(attributes::common::cluster_name, cluster_name.value());
   }
   if (cluster_uuid) {
-    span->add_tag(attributes::cluster_uuid, cluster_uuid.value());
+    span->add_tag(attributes::common::cluster_uuid, cluster_uuid.value());
   }
 
   return span;
 }
 
-void
-tracer_wrapper::update_config(topology::configuration config)
+auto
+tracer_wrapper::wrapped() -> std::shared_ptr<couchbase::tracing::request_tracer>
 {
-  const std::scoped_lock<std::shared_mutex> lock{ cluster_labels_mutex_ };
-  if (config.cluster_uuid.has_value()) {
-    cluster_uuid_ = config.cluster_uuid;
-  }
-  if (config.cluster_name.has_value()) {
-    cluster_name_ = config.cluster_name;
-  }
+  return tracer_;
 }
 
 auto
-tracer_wrapper::create(std::shared_ptr<couchbase::tracing::request_tracer> tracer)
+tracer_wrapper::create(std::shared_ptr<couchbase::tracing::request_tracer> tracer,
+                       std::shared_ptr<cluster_label_listener> label_listener)
   -> std::shared_ptr<tracer_wrapper>
 {
-  return std::make_shared<tracer_wrapper>(std::move(tracer));
+  return std::make_shared<tracer_wrapper>(std::move(tracer), std::move(label_listener));
 }
 } // namespace couchbase::core::tracing

data/ext/couchbase/core/tracing/tracer_wrapper.hxx

@@ -17,8 +17,7 @@
 
 #pragma once
 
-#include "core/config_listener.hxx"
-#include "core/topology/configuration.hxx"
+#include "core/cluster_label_listener.hxx"
 
 #include <couchbase/tracing/request_span.hxx>
 #include <couchbase/tracing/request_tracer.hxx>
@@ -30,10 +29,11 @@
 
 namespace couchbase::core::tracing
 {
-class tracer_wrapper : public config_listener
+class tracer_wrapper
 {
 public:
-  explicit tracer_wrapper(std::shared_ptr<couchbase::tracing::request_tracer> tracer);
+  explicit tracer_wrapper(std::shared_ptr<couchbase::tracing::request_tracer> tracer,
+                          std::shared_ptr<cluster_label_listener> label_listener);
 
   void start();
   void stop();
@@ -42,16 +42,14 @@ public:
                    std::shared_ptr<couchbase::tracing::request_span> parent_span)
     -> std::shared_ptr<couchbase::tracing::request_span>;
 
-  void update_config(topology::configuration config) override;
+  [[nodiscard]] auto wrapped() -> std::shared_ptr<couchbase::tracing::request_tracer>;
 
-  [[nodiscard]] static auto create(std::shared_ptr<couchbase::tracing::request_tracer> tracer)
+  [[nodiscard]] static auto create(std::shared_ptr<couchbase::tracing::request_tracer> tracer,
+                                   std::shared_ptr<cluster_label_listener> label_listener)
     -> std::shared_ptr<tracer_wrapper>;
 
 private:
   std::shared_ptr<couchbase::tracing::request_tracer> tracer_;
-
-  std::optional<std::string> cluster_name_{};
-  std::optional<std::string> cluster_uuid_{};
-  std::shared_mutex cluster_labels_mutex_{};
+  std::shared_ptr<couchbase::core::cluster_label_listener> cluster_label_listener_;
 };
 } // namespace couchbase::core::tracing

data/ext/couchbase/core/tracing/wrapper_sdk_tracer.cxx

@@ -0,0 +1,114 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2025. Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "wrapper_sdk_tracer.hxx"
+
+#include "core/logger/logger.hxx"
+
+namespace couchbase::core::tracing
+{
+auto
+wrapper_sdk_tracer::start_span(std::string name,
+                               std::shared_ptr<couchbase::tracing::request_span> parent)
+  -> std::shared_ptr<couchbase::tracing::request_span>
+{
+  const auto parent_wrapper_span = std::dynamic_pointer_cast<wrapper_sdk_span>(parent);
+  if (!parent_wrapper_span) {
+    // If no parent span is provided, wrappers have no way of accessing any child spans created, so
+    // there is no reason to create spans.
+    return noop_instance_;
+  }
+  const auto span = std::make_shared<wrapper_sdk_span>(std::move(name), std::move(parent));
+  parent_wrapper_span->add_child(span);
+  return span;
+}
+
+wrapper_sdk_span::wrapper_sdk_span(std::string name)
+  : couchbase::tracing::request_span{ std::move(name) }
+{
+}
+
+wrapper_sdk_span::wrapper_sdk_span(std::string name,
+                                   const std::shared_ptr<couchbase::tracing::request_span>& parent)
+  : couchbase::tracing::request_span{ std::move(name) }
+  , parent_{ parent }
+{
+}
+
+void
+wrapper_sdk_span::add_child(const std::shared_ptr<wrapper_sdk_span>& child)
+{
+  const std::scoped_lock lock(children_mutex_);
+  children_.emplace_back(child);
+}
+
+void
+wrapper_sdk_span::add_tag(const std::string& name, std::uint64_t value)
+{
+  uint_tags_.emplace(name, value);
+}
+
+void
+wrapper_sdk_span::add_tag(const std::string& name, const std::string& value)
+{
+  string_tags_.emplace(name, value);
+}
+
+void
+wrapper_sdk_span::end()
+{
+  end_time_ = std::chrono::system_clock::now();
+}
+
+auto
+wrapper_sdk_span::uint_tags() const -> const std::map<std::string, std::uint64_t>&
+{
+  return uint_tags_;
+}
+
+auto
+wrapper_sdk_span::string_tags() const -> const std::map<std::string, std::string>&
+{
+  return string_tags_;
+}
+
+auto
+wrapper_sdk_span::children() -> std::vector<std::shared_ptr<wrapper_sdk_span>>
+{
+  const std::scoped_lock lock(children_mutex_);
+  return children_;
+}
+
+auto
+wrapper_sdk_span::start_time() const -> const std::chrono::system_clock::time_point&
+{
+  return start_time_;
+}
+
+auto
+wrapper_sdk_span::end_time() const -> const std::chrono::system_clock::time_point&
+{
+  return end_time_;
+}
+
+auto
+wrapper_sdk_span::parent() const -> std::shared_ptr<couchbase::tracing::request_span>
+{
+  return parent_.lock();
+}
+
+} // namespace couchbase::core::tracing