couchbase 3.7.0 → 3.8.0

data/lib/couchbase/tracing/threshold_logging_tracer.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+#  Copyright 2025-Present Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require 'couchbase/tracing/request_tracer'
+require 'couchbase/tracing/threshold_logging_span'
+require 'couchbase/utils/observability_constants'
+require 'couchbase/logger'
+require 'couchbase/utils/stdlib_logger_adapter'
+
+require 'json'
+
+module Couchbase
+  module Tracing
+    class ThresholdLoggingTracer < RequestTracer
+      # @api private
+      DEFAULT_KV_THRESHOLD = 500 # milliseconds
+
+      # @api private
+      DEFAULT_HTTP_THRESHOLD = 1_000 # milliseconds
+
+      # @api private
+      DEFAULT_SAMPLE_SIZE = 10
+
+      # @api private
+      DEFAULT_EMIT_INTERVAL = 10_000 # milliseconds
+
+      def initialize(
+        emit_interval: nil,
+        kv_threshold: nil,
+        query_threshold: nil,
+        views_threshold: nil,
+        search_threshold: nil,
+        analytics_threshold: nil,
+        management_threshold: nil,
+        sample_size: nil
+      )
+        super()
+
+        emit_interval = DEFAULT_EMIT_INTERVAL if emit_interval.nil?
+        kv_threshold = DEFAULT_KV_THRESHOLD if kv_threshold.nil?
+        query_threshold = DEFAULT_HTTP_THRESHOLD if query_threshold.nil?
+        views_threshold = DEFAULT_HTTP_THRESHOLD if views_threshold.nil?
+        search_threshold = DEFAULT_HTTP_THRESHOLD if search_threshold.nil?
+        analytics_threshold = DEFAULT_HTTP_THRESHOLD if analytics_threshold.nil?
+        management_threshold = DEFAULT_HTTP_THRESHOLD if management_threshold.nil?
+        sample_size = DEFAULT_SAMPLE_SIZE if sample_size.nil?
+
+        raise ArgumentError, "The sample size for ThresholdLoggingTracer must be positive" unless sample_size.positive?
+
+        @emit_interval = emit_interval
+        @sample_size = sample_size
+        @groups = {
+          Observability::ATTR_VALUE_SERVICE_KV => Group.new(floor_us: 1000 * kv_threshold, capacity: @sample_size),
+          Observability::ATTR_VALUE_SERVICE_QUERY => Group.new(floor_us: 1000 * query_threshold, capacity: @sample_size),
+          Observability::ATTR_VALUE_SERVICE_VIEWS => Group.new(floor_us: 1000 * views_threshold, capacity: @sample_size),
+          Observability::ATTR_VALUE_SERVICE_SEARCH => Group.new(floor_us: 1000 * search_threshold, capacity: @sample_size),
+          Observability::ATTR_VALUE_SERVICE_ANALYTICS => Group.new(floor_us: 1000 * analytics_threshold, capacity: @sample_size),
+          Observability::ATTR_VALUE_SERVICE_MANAGEMENT => Group.new(floor_us: 1000 * management_threshold, capacity: @sample_size),
+        }
+
+        # TODO(DC): Find better solution for logging
+        @logger = Couchbase.logger || Logger.new($stdout, Utils::StdlibLoggerAdapter.map_spdlog_level(Couchbase.log_level))
+        start_reporting_thread
+      end
+
+      def request_span(name, parent: nil, start_timestamp: nil)
+        ThresholdLoggingSpan.new(
+          name,
+          start_timestamp: start_timestamp,
+          parent: parent,
+          tracer: self,
+        )
+      end
+
+      def record_operation(service, item)
+        group = @groups[service]
+        return if group.nil?
+
+        group.record_operation(item)
+      end
+
+      def close
+        @thread.exit
+      end
+
+      def create_report
+        data = {}
+        @groups.each do |service, group|
+          group_data = group.steal_data
+          next if group_data.nil?
+
+          data[service] = group_data
+        end
+        data
+      end
+
+      def start_reporting_thread
+        @thread = Thread.new do # rubocop:disable ThreadSafety/NewThread
+          loop do
+            sleep(@emit_interval / 1_000.0)
+            report = create_report
+
+            next if report.empty?
+
+            begin
+              @logger.info("Threshold Logging Report: #{report.to_json}")
+            rescue StandardError => e
+              @logger.debug("Failed to log threshold logging report: #{e.message}")
+            end
+          end
+        end
+      end
+
+      class Item
+        attr_accessor :total_duration_us
+        attr_accessor :encode_duration_us
+        attr_accessor :last_dispatch_duration_us
+        attr_accessor :total_dispatch_duration_us
+        attr_accessor :last_server_duration_us
+        attr_accessor :total_server_duration_us
+        attr_accessor :operation_name
+        attr_accessor :last_local_id
+        attr_accessor :operation_id
+        attr_accessor :last_remote_socket
+
+        def initialize(
+          total_duration_us:,
+          encode_duration_us:,
+          last_dispatch_duration_us:,
+          total_dispatch_duration_us:,
+          last_server_duration_us:,
+          total_server_duration_us:,
+          operation_name:,
+          last_local_id:,
+          operation_id:,
+          last_remote_socket:
+        )
+          @total_duration_us = total_duration_us
+          @encode_duration_us = encode_duration_us
+          @last_dispatch_duration_us = last_dispatch_duration_us
+          @total_dispatch_duration_us = total_dispatch_duration_us
+          @last_server_duration_us = last_server_duration_us
+          @total_server_duration_us = total_server_duration_us
+          @operation_name = operation_name
+          @last_local_id = last_local_id
+          @operation_id = operation_id
+          @last_remote_socket = last_remote_socket
+        end
+
+        def to_h
+          {
+            total_duration_us: @total_duration_us,
+            encode_duration_us: @encode_duration_us,
+            last_dispatch_duration_us: @last_dispatch_duration_us,
+            total_dispatch_duration_us: @total_dispatch_duration_us,
+            last_server_duration_us: @last_server_duration_us,
+            total_server_duration_us: @total_server_duration_us,
+            operation_name: @operation_name,
+            last_local_id: @last_local_id,
+            operation_id: @operation_id,
+            last_remote_socket: @last_remote_socket,
+          }.compact
+        end
+      end
+
+      class Group
+        def initialize(capacity:, floor_us:)
+          @capacity = capacity
+          @floor_us = floor_us
+
+          @total_count = 0
+          @top_requests = []
+          @mutex = Mutex.new
+        end
+
+        def record_operation(item)
+          return if item.total_duration_us < @floor_us
+
+          @mutex.synchronize do
+            @total_count += 1
+
+            return if @top_requests.size >= @capacity && item.total_duration_us < @top_requests[-1].total_duration_us
+
+            idx = @top_requests.bsearch_index do |x|
+              item.total_duration_us >= x.total_duration_us
+            end
+
+            # The item is smaller than all existing items. We will insert it at the end.
+            idx = @top_requests.size if idx.nil?
+
+            if @top_requests.size >= @capacity
+              # We are at capacity, remove the smallest (last) item
+              @top_requests.pop
+            end
+            @top_requests.insert(idx, item)
+          end
+        end
+
+        def steal_data
+          top_requests, total_count = @mutex.synchronize do
+            top_requests_tmp = @top_requests
+            total_count_tmp = @total_count
+            @top_requests = []
+            @total_count = 0
+            [top_requests_tmp, total_count_tmp]
+          end
+
+          return nil if total_count.zero?
+
+          {
+            total_count: total_count,
+            top_requests: top_requests.map(&:to_h),
+          }
+        end
+      end
+    end
+  end
+end

data/lib/couchbase/utils/hdr_histogram.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+#  Copyright 2025-Present Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+module Couchbase
+  module Utils
+    class HdrHistogram
+      def initialize(
+        lowest_discernible_value:,
+        highest_trackable_value:,
+        significant_figures:,
+        percentiles: nil
+      )
+        @histogram_backend = HdrHistogramC.new(lowest_discernible_value, highest_trackable_value, significant_figures)
+        @percentiles = percentiles || [50.0, 90.0, 99.0, 99.9, 100.0]
+      end
+
+      def record_value(value)
+        @histogram_backend.record_value(value)
+      end
+
+      def close
+        @histogram_backend.close
+      end
+
+      def report_and_reset
+        backend_report = @histogram_backend.get_percentiles_and_reset(@percentiles)
+        total_count = backend_report[:total_count]
+
+        return nil if total_count.zero?
+
+        report = {
+          total_count: total_count,
+          percentiles_us: {},
+        }
+        @percentiles.zip(backend_report[:percentiles]).each do |percentile, percentile_value|
+          report[:percentiles_us][percentile.to_s] = percentile_value
+        end
+        report
+      end
+    end
+  end
+end

data/lib/couchbase/utils/observability.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+#  Copyright 2025-Present Couchbase, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require "couchbase/tracing/request_span"
+require "couchbase/tracing/request_tracer"
+
+require_relative "observability_constants"
+
+module Couchbase
+  module Observability
+    class ClusterLabelListener
+      attr_reader :cluster_name
+      attr_reader :cluster_uuid
+    end
+
+    class Wrapper
+      attr_accessor :tracer
+      attr_accessor :meter
+
+      def initialize(backend:, tracer: nil, meter: nil)
+        @backend = backend
+        @tracer = tracer
+        @meter = meter
+
+        yield self if block_given?
+      end
+
+      def record_operation(op_name, parent_span, receiver, service = nil)
+        handler = Handler.new(@backend, op_name, parent_span, receiver, @tracer, @meter)
+        handler.add_operation_name(op_name)
+        handler.add_service(service) unless service.nil?
+        begin
+          res = yield(handler)
+        rescue StandardError => e
+          handler.add_error(e)
+          raise e
+        else
+          handler.set_success
+        ensure
+          handler.finish
+        end
+        res
+      end
+
+      def close
+        @tracer&.close
+        @meter&.close
+      end
+    end
+
+    class Handler
+      attr_reader :op_span
+
+      def initialize(backend, op_name, parent_span, receiver, tracer, meter)
+        @tracer = tracer
+        @meter = meter
+
+        cluster_labels = backend.cluster_labels
+        @cluster_name = cluster_labels[:cluster_name]
+        @cluster_uuid = cluster_labels[:cluster_uuid]
+
+        @op_span = create_span(op_name, parent_span)
+        @meter_attributes = create_meter_attributes
+        @start_time = Time.now
+        add_operation_name(op_name)
+        add_receiver_attributes(receiver)
+      end
+
+      def with_request_encoding_span
+        span = create_span(STEP_REQUEST_ENCODING, @op_span)
+        begin
+          res = yield
+        ensure
+          span.finish
+        end
+        res
+      end
+
+      def add_service(service)
+        service_str =
+          case service
+          when :kv
+            ATTR_VALUE_SERVICE_KV
+          when :query
+            ATTR_VALUE_SERVICE_QUERY
+          when :analytics
+            ATTR_VALUE_SERVICE_ANALYTICS
+          when :search
+            ATTR_VALUE_SERVICE_SEARCH
+          when :management
+            ATTR_VALUE_SERVICE_MANAGEMENT
+          when :views
+            ATTR_VALUE_SERVICE_VIEWS
+          end
+        @op_span.set_attribute(ATTR_SERVICE, service_str) unless service_str.nil?
+        @meter_attributes[ATTR_SERVICE] = service_str unless service_str.nil?
+      end
+
+      def add_operation_name(name)
+        @op_span.set_attribute(ATTR_OPERATION_NAME, name)
+        @meter_attributes[ATTR_OPERATION_NAME] = name
+      end
+
+      def add_bucket_name(name)
+        @op_span.set_attribute(ATTR_BUCKET_NAME, name)
+        @meter_attributes[ATTR_BUCKET_NAME] = name
+      end
+
+      def add_scope_name(name)
+        @op_span.set_attribute(ATTR_SCOPE_NAME, name)
+        @meter_attributes[ATTR_SCOPE_NAME] = name
+      end
+
+      def add_collection_name(name)
+        @op_span.set_attribute(ATTR_COLLECTION_NAME, name)
+        @meter_attributes[ATTR_COLLECTION_NAME] = name
+      end
+
+      def add_durability_level(level)
+        durability_str =
+          case level
+          when :majority
+            ATTR_VALUE_DURABILITY_MAJORITY
+          when :majority_and_persist_to_active
+            ATTR_VALUE_DURABILITY_MAJORITY_AND_PERSIST_TO_ACTIVE
+          when :persist_to_majority
+            ATTR_VALUE_DURABILITY_PERSIST_TO_MAJORITY
+          end
+        @op_span.set_attribute(ATTR_DURABILITY, durability_str) unless durability_str.nil?
+      end
+
+      def add_retries(retries)
+        @op_span.set_attribute(ATTR_RETRIES, retries.to_i)
+      end
+
+      def set_success
+        @op_span.status = :ok
+      end
+
+      def add_error(error)
+        @op_span.status = :error
+        @meter_attributes[ATTR_ERROR_TYPE] =
+          if error.is_a?(Couchbase::Error::CouchbaseError) || error.is_a?(Couchbase::Error::InvalidArgument)
+            error.class.name.split("::").last
+          else
+            "_OTHER"
+          end
+      end
+
+      def add_query_statement(statement, options)
+        pos_params = options.instance_variable_get(:@positional_parameters)
+        named_params = options.instance_variable_get(:@named_parameters)
+
+        # The statement attribute is added only if positional or named parameters are in use.
+        return if (pos_params.nil? || pos_params.empty?) && (named_params.nil? || named_params.empty?)
+
+        @op_span.set_attribute(ATTR_QUERY_STATEMENT, statement)
+      end
+
+      def add_spans_from_backend(backend_spans)
+        backend_spans.each do |backend_span|
+          add_backend_span(backend_span, @op_span)
+        end
+      end
+
+      def finish
+        @op_span.finish
+        duration_us = ((Time.now - @start_time) * 1_000_000).round
+        @meter.value_recorder(METER_NAME_OPERATION_DURATION, @meter_attributes).record_value(duration_us)
+      end
+
+      private
+
+      def add_backend_span(backend_span, parent)
+        span = @tracer.request_span(
+          backend_span[:name],
+          parent: parent,
+          start_timestamp: convert_backend_timestamp(backend_span[:start_timestamp]),
+        )
+        backend_span[:attributes].each do |k, v|
+          span.set_attribute(k, v)
+        end
+        if backend_span.key?(:children)
+          backend_span[:children].each do |c|
+            add_backend_span(c, span)
+          end
+        end
+        span.finish(end_timestamp: convert_backend_timestamp(backend_span[:end_timestamp]))
+      end
+
+      def convert_backend_timestamp(backend_timestamp)
+        Time.at(backend_timestamp / (10**6), backend_timestamp % (10**6))
+      end
+
+      def create_meter_attributes
+        attrs = {
+          ATTR_SYSTEM_NAME => ATTR_VALUE_SYSTEM_NAME,
+          ATTR_RESERVED_UNIT => ATTR_VALUE_RESERVED_UNIT_SECONDS,
+        }
+        attrs[ATTR_CLUSTER_NAME] = @cluster_name unless @cluster_name.nil?
+        attrs[ATTR_CLUSTER_UUID] = @cluster_uuid unless @cluster_uuid.nil?
+        attrs
+      end
+
+      def create_span(name, parent, start_timestamp: nil)
+        span = @tracer.request_span(name, parent: parent, start_timestamp: start_timestamp)
+        span.set_attribute(ATTR_SYSTEM_NAME, ATTR_VALUE_SYSTEM_NAME)
+        span.set_attribute(ATTR_CLUSTER_NAME, @cluster_name) unless @cluster_name.nil?
+        span.set_attribute(ATTR_CLUSTER_UUID, @cluster_uuid) unless @cluster_uuid.nil?
+        span
+      end
+
+      def add_receiver_attributes(receiver)
+        if receiver.instance_variable_defined?(:@collection)
+          collection = receiver.instance_variable_get(:@collection)
+          add_bucket_name(collection.bucket_name)
+          add_scope_name(collection.scope_name)
+          add_collection_name(collection.name)
+          return
+        end
+
+        if receiver.instance_variable_defined?(:@bucket_name)
+          add_bucket_name(receiver.instance_variable_get(:@bucket_name))
+        elsif receiver.instance_variable_defined?(:@name)
+          add_bucket_name(receiver.instance_variable_get(:@name))
+          return
+        end
+
+        if receiver.instance_variable_defined?(:@scope_name)
+          add_scope_name(receiver.instance_variable_get(:@scope_name))
+        elsif receiver.instance_variable_defined?(:@name)
+          add_scope_name(receiver.instance_variable_get(:@name))
+          return
+        end
+
+        if receiver.instance_variable_defined?(:@collection_name)
+          add_collection_name(receiver.instance_variable_get(:@collection_name))
+        elsif receiver.instance_variable_defined?(:@name)
+          add_collection_name(receiver.instance_variable_get(:@name))
+        end
+      end
+    end
+  end
+end