ddtrace 0.54.2 → 1.0.0.beta1

data/lib/datadog/profiling/transport/http/api.rb

@@ -0,0 +1,44 @@
+# typed: true
+require 'ddtrace/transport/http/api/map'
+require 'datadog/profiling/encoding/profile'
+require 'datadog/profiling/transport/http/api/spec'
+require 'datadog/profiling/transport/http/api/instance'
+require 'datadog/profiling/transport/http/api/endpoint'
+
+module Datadog
+  module Profiling
+    module Transport
+      module HTTP
+        # Extensions for HTTP API Spec
+        module API
+          # Default API versions
+          V1 = 'v1'.freeze
+
+          module_function
+
+          def agent_defaults
+            @agent_defaults ||= Datadog::Transport::HTTP::API::Map[
+              V1 => Spec.new do |s|
+                s.profiles = Endpoint.new(
+                  '/profiling/v1/input'.freeze,
+                  Profiling::Encoding::Profile::Protobuf
+                )
+              end
+            ]
+          end
+
+          def api_defaults
+            @api_defaults ||= Datadog::Transport::HTTP::API::Map[
+              V1 => Spec.new do |s|
+                s.profiles = Endpoint.new(
+                  '/v1/input'.freeze,
+                  Profiling::Encoding::Profile::Protobuf
+                )
+              end
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/http/builder.rb

@@ -0,0 +1,29 @@
+# typed: true
+require 'ddtrace/transport/http/builder'
+
+require 'datadog/profiling/transport/http/api'
+require 'datadog/profiling/transport/http/client'
+
+module Datadog
+  module Profiling
+    module Transport
+      module HTTP
+        # Builds new instances of Transport::HTTP::Client
+        class Builder < Datadog::Transport::HTTP::Builder
+          def api_instance_class
+            API::Instance
+          end
+
+          def to_transport
+            raise Datadog::Transport::HTTP::Builder::NoDefaultApiError if @default_api.nil?
+
+            # TODO: Profiling doesn't have multiple APIs yet.
+            #       When it does, we should build it out with these APIs.
+            #       Just use :default_api for now.
+            Client.new(to_api_instances[@default_api])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/http/client.rb

@@ -0,0 +1,34 @@
+# typed: true
+require 'ddtrace/transport/http/client'
+require 'datadog/profiling/transport/client'
+
+module Datadog
+  module Profiling
+    module Transport
+      module HTTP
+        # Routes, encodes, and sends tracer data to the trace agent via HTTP.
+        class Client < Datadog::Transport::HTTP::Client
+          include Transport::Client
+
+          def send_profiling_flush(flush)
+            # Build a request
+            request = Profiling::Transport::Request.new(flush)
+            send_payload(request).tap do |response|
+              if response.ok?
+                Datadog.logger.debug('Successfully reported profiling data')
+              else
+                Datadog.logger.debug { "Failed to report profiling data -- #{response.inspect}" }
+              end
+            end
+          end
+
+          def send_payload(request)
+            send_request(request) do |api, env|
+              api.send_profiling_flush(env)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/http/response.rb

@@ -0,0 +1,22 @@
+# typed: true
+require 'ddtrace/transport/http/response'
+require 'datadog/profiling/transport/response'
+
+module Datadog
+  module Profiling
+    module Transport
+      # HTTP transport behavior for profiling
+      module HTTP
+        # Response from HTTP transport for profiling
+        class Response
+          include Datadog::Transport::HTTP::Response
+          include Profiling::Transport::Response
+
+          def initialize(http_response, options = {})
+            super(http_response)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/http.rb

@@ -0,0 +1,111 @@
+# typed: true
+require 'datadog/core/environment/ext'
+require 'ddtrace/transport/ext'
+
+require 'datadog/core/environment/container'
+require 'datadog/core/environment/variable_helpers'
+
+require 'datadog/profiling/transport/http/builder'
+require 'datadog/profiling/transport/http/api'
+
+require 'ddtrace/transport/http/adapters/net'
+require 'ddtrace/transport/http/adapters/test'
+require 'ddtrace/transport/http/adapters/unix_socket'
+
+module Datadog
+  module Profiling
+    module Transport
+      # Namespace for HTTP transport components
+      module HTTP
+        # Builds a new Transport::HTTP::Client
+        def self.new(&block)
+          Builder.new(&block).to_transport
+        end
+
+        # Builds a new Transport::HTTP::Client with default settings
+        def self.default(
+          profiling_upload_timeout_seconds:,
+          agent_settings:,
+          site: nil,
+          api_key: nil,
+          agentless_allowed: agentless_allowed?
+        )
+          new do |transport|
+            transport.headers default_headers
+
+            # Configure adapter & API
+            if site && api_key && agentless_allowed
+              configure_for_agentless(
+                transport,
+                profiling_upload_timeout_seconds: profiling_upload_timeout_seconds,
+                site: site,
+                api_key: api_key
+              )
+            else
+              configure_for_agent(
+                transport,
+                profiling_upload_timeout_seconds: profiling_upload_timeout_seconds,
+                agent_settings: agent_settings
+              )
+            end
+          end
+        end
+
+        def self.default_headers
+          {
+            Datadog::Transport::Ext::HTTP::HEADER_META_LANG => Core::Environment::Ext::LANG,
+            Datadog::Transport::Ext::HTTP::HEADER_META_LANG_VERSION => Core::Environment::Ext::LANG_VERSION,
+            Datadog::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER => Core::Environment::Ext::LANG_INTERPRETER,
+            Datadog::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION => Core::Environment::Ext::TRACER_VERSION
+          }.tap do |headers|
+            # Add container ID, if present.
+            container_id = Core::Environment::Container.container_id
+            headers[Datadog::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
+          end
+        end
+
+        private_class_method def self.configure_for_agent(transport, profiling_upload_timeout_seconds:, agent_settings:)
+          apis = API.agent_defaults
+
+          transport.adapter(agent_settings.merge(timeout_seconds: profiling_upload_timeout_seconds))
+          transport.api(API::V1, apis[API::V1], default: true)
+
+          # NOTE: This proc, when it exists, usually overrides the transport specified above
+          if agent_settings.deprecated_for_removal_transport_configuration_proc
+            agent_settings.deprecated_for_removal_transport_configuration_proc.call(transport)
+          end
+        end
+
+        private_class_method def self.configure_for_agentless(transport, profiling_upload_timeout_seconds:, site:, api_key:)
+          apis = API.api_defaults
+
+          site_uri = URI(format(Profiling::Ext::Transport::HTTP::URI_TEMPLATE_DD_API, site))
+          hostname = site_uri.host
+          port = site_uri.port
+
+          transport.adapter(
+            Datadog::Transport::Ext::HTTP::ADAPTER,
+            hostname,
+            port,
+            timeout: profiling_upload_timeout_seconds,
+            ssl: site_uri.scheme == 'https'
+          )
+          transport.api(API::V1, apis[API::V1], default: true)
+          transport.headers(Datadog::Transport::Ext::HTTP::HEADER_DD_API_KEY => api_key)
+        end
+
+        private_class_method def self.agentless_allowed?
+          Core::Environment::VariableHelpers.env_to_bool(Profiling::Ext::ENV_AGENTLESS, false)
+        end
+
+        # Add adapters to registry
+        Datadog::Transport::HTTP::Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::Net,
+                                                        Datadog::Transport::Ext::HTTP::ADAPTER)
+        Datadog::Transport::HTTP::Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::Test,
+                                                        Datadog::Transport::Ext::Test::ADAPTER)
+        Datadog::Transport::HTTP::Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::UnixSocket,
+                                                        Datadog::Transport::Ext::UnixSocket::ADAPTER)
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/io/client.rb

@@ -0,0 +1,28 @@
+# typed: true
+require 'ddtrace/transport/io/client'
+require 'datadog/profiling/transport/client'
+require 'datadog/profiling/transport/request'
+require 'datadog/profiling/transport/io/response'
+
+module Datadog
+  module Profiling
+    module Transport
+      module IO
+        # IO transport for profiling
+        class Client < Datadog::Transport::IO::Client
+          include Transport::Client
+
+          def send_profiling_flush(flush)
+            # Build a request
+            request = Profiling::Transport::Request.new(flush)
+            send_request(request)
+          end
+
+          def build_response(_request, _data, result)
+            Profiling::Transport::IO::Response.new(result)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/io/response.rb

@@ -0,0 +1,17 @@
+# typed: strict
+require 'ddtrace/transport/io/response'
+require 'datadog/profiling/transport/response'
+
+module Datadog
+  module Profiling
+    module Transport
+      # IO transport behavior for profiling
+      module IO
+        # Response from IO transport for profiling
+        class Response < Datadog::Transport::IO::Response
+          include Profiling::Transport::Response
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/io.rb

@@ -0,0 +1,31 @@
+# typed: true
+require 'datadog/profiling/transport/io/client'
+require 'datadog/profiling/encoding/profile'
+
+module Datadog
+  module Profiling
+    module Transport
+      # Namespace for profiling IO transport components
+      module IO
+        module_function
+
+        # Builds a new Profiling::Transport::IO::Client
+        def new(out, encoder, options = {})
+          Client.new(out, encoder, options)
+        end
+
+        # Builds a new Profiling::Transport::IO::Client with default settings
+        # Pass options to override any settings.
+        def default(options = {})
+          options = options.dup
+
+          new(
+            options.delete(:out) || $stdout,
+            options.delete(:encoder) || Profiling::Encoding::Profile::Protobuf,
+            options
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/transport/request.rb

@@ -0,0 +1,16 @@
+# typed: true
+require 'ddtrace/transport/request'
+require 'datadog/profiling/transport/parcel'
+
+module Datadog
+  module Profiling
+    module Transport
+      # Profiling request
+      class Request < Datadog::Transport::Request
+        def initialize(flush)
+          super(Parcel.new(flush))
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling.rb

@@ -0,0 +1,167 @@
+# typed: true
+require 'datadog/core'
+require 'datadog/core/environment/variable_helpers'
+require 'datadog/core/utils/only_once'
+
+module Datadog
+  # Contains profiler for generating stack profiles, etc.
+  module Profiling # rubocop:disable Metrics/ModuleLength
+    GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
+    private_constant :GOOGLE_PROTOBUF_MINIMUM_VERSION
+
+    SKIPPED_NATIVE_EXTENSION_ONLY_ONCE = Core::Utils::OnlyOnce.new
+    private_constant :SKIPPED_NATIVE_EXTENSION_ONLY_ONCE
+
+    def self.supported?
+      unsupported_reason.nil?
+    end
+
+    def self.unsupported_reason
+      # NOTE: Only the first matching reason is returned, so try to keep a nice order on reasons -- e.g. tell users
+      # first that they can't use this on JRuby before telling them that they are missing protobuf
+
+      ruby_engine_unsupported? ||
+        native_library_failed_to_load? ||
+        protobuf_gem_unavailable? ||
+        protobuf_version_unsupported? ||
+        protobuf_failed_to_load?
+    end
+
+    # Starts the profiler, if the profiler is supported by in
+    # this runtime environment and if the profiler has been enabled
+    # in configuration.
+    #
+    # @return [Boolean] `true` if the profiler has successfully started, otherwise `false`.
+    # @public_api
+    def self.start_if_enabled
+      # If the profiler was not previously touched, getting the profiler instance triggers start as a side-effect
+      # otherwise we get nil
+      profiler = Datadog.send(:components).profiler
+      # ...but we still try to start it BECAUSE if the process forks, the profiler will exist but may
+      # not yet have been started in the fork
+      profiler.start if profiler
+      !!profiler
+    end
+
+    private_class_method def self.ruby_engine_unsupported?
+      'JRuby is not supported' if RUBY_ENGINE == 'jruby'
+    end
+
+    private_class_method def self.protobuf_gem_unavailable?
+      # NOTE: On environments where protobuf is already loaded, we skip the check. This allows us to support environments
+      # where no Gem.loaded_version is NOT available but customers are able to load protobuf; see for instance
+      # https://github.com/teamcapybara/capybara/commit/caf3bcd7664f4f2691d0ca9ef3be9a2a954fecfb
+      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].nil?
+        "Missing google-protobuf dependency; please add `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
+      end
+    end
+
+    private_class_method def self.protobuf_version_unsupported?
+      # See above for why we skip the check when protobuf is already loaded; note that when protobuf was already loaded
+      # we skip the version check to avoid the call to Gem.loaded_specs. Unfortunately, protobuf does not seem to
+      # expose the gem version constant elsewhere, so in that setup we are not able to check the version.
+      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
+        'Your google-protobuf is too old; ensure that you have google-protobuf >= 3.0 by ' \
+        "adding `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
+      end
+    end
+
+    private_class_method def self.protobuf_failed_to_load?
+      unless protobuf_loaded_successfully?
+        'There was an error loading the google-protobuf library; see previous warning message for details'
+      end
+    end
+
+    # The `google-protobuf` gem depends on a native component, and its creators helpfully tried to provide precompiled
+    # versions of this extension on rubygems.org.
+    #
+    # Unfortunately, for a long time, the supported Ruby versions metadata on these precompiled versions of the extension
+    # was not correctly set. (This is fixed in newer versions -- but not all Ruby versions we want to support can use
+    # these.)
+    #
+    # Thus, the gem can still be installed, but can be in a broken state. To avoid breaking customer applications, we
+    # use this helper to load it and gracefully handle failures.
+    private_class_method def self.protobuf_loaded_successfully?
+      return @protobuf_loaded if defined?(@protobuf_loaded)
+
+      begin
+        require 'google/protobuf'
+        @protobuf_loaded = true
+      rescue LoadError => e
+        # NOTE: We use Kernel#warn here because this code gets run BEFORE Datadog.logger is actually set up.
+        # In the future it'd be nice to shuffle the logger startup to happen first to avoid this special case.
+        Kernel.warn(
+          '[DDTRACE] Error while loading google-protobuf gem. ' \
+          "Cause: '#{e.message}' Location: '#{Array(e.backtrace).first}'. " \
+          'This can happen when google-protobuf is missing its native components. ' \
+          'To fix this, try removing and reinstalling the gem, forcing it to recompile the components: ' \
+          '`gem uninstall google-protobuf -a; BUNDLE_FORCE_RUBY_PLATFORM=true bundle install`. ' \
+          'If the error persists, please contact support via <https://docs.datadoghq.com/help/> or ' \
+          'file a bug at <https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug>.'
+        )
+        @protobuf_loaded = false
+      end
+    end
+
+    private_class_method def self.native_library_failed_to_load?
+      success, exception = try_loading_native_library
+
+      unless success
+        if exception
+          'There was an error loading the profiling native extension due to ' \
+          "'#{exception.message}' at '#{exception.backtrace.first}'"
+        else
+          'The profiling native extension did not load correctly. ' \
+          'If the error persists, please contact support via <https://docs.datadoghq.com/help/> or ' \
+          'file a bug at <https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug>.'
+        end
+      end
+    end
+
+    private_class_method def self.try_loading_native_library
+      if Core::Environment::VariableHelpers.env_to_bool('DD_PROFILING_NO_EXTENSION', false)
+        SKIPPED_NATIVE_EXTENSION_ONLY_ONCE.run do
+          Kernel.warn(
+            '[DDTRACE] Skipped loading of profiling native extension due to DD_PROFILING_NO_EXTENSION environment ' \
+            'variable being set. ' \
+            'This option is experimental and will lead to the profiler not working in future releases. ' \
+            'If you needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.'
+          )
+        end
+
+        return [true, nil]
+      end
+
+      begin
+        require "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+        success =
+          defined?(Profiling::NativeExtension) && Profiling::NativeExtension.send(:native_working?)
+        [success, nil]
+      rescue StandardError, LoadError => e
+        [false, e]
+      end
+    end
+
+    private_class_method def self.load_profiling
+      return false unless supported?
+
+      require 'datadog/profiling/ext/forking'
+      require 'datadog/profiling/collectors/code_provenance'
+      require 'datadog/profiling/collectors/stack'
+      require 'datadog/profiling/exporter'
+      require 'datadog/profiling/recorder'
+      require 'datadog/profiling/scheduler'
+      require 'datadog/profiling/tasks/setup'
+      require 'datadog/profiling/transport/io'
+      require 'datadog/profiling/transport/http'
+      require 'datadog/profiling/profiler'
+      require 'datadog/profiling/native_extension'
+      require 'datadog/profiling/trace_identifiers/helper'
+      require 'datadog/profiling/pprof/pprof_pb'
+
+      true
+    end
+
+    load_profiling if supported?
+  end
+end

data/lib/datadog/tracing/analytics.rb

@@ -0,0 +1,24 @@
+# typed: true
+require 'datadog/tracing/metadata/ext'
+
+module Datadog
+  module Tracing
+    # Defines analytics behavior
+    module Analytics
+      class << self
+        def set_sample_rate(span_op, sample_rate)
+          return if span_op.nil? || !sample_rate.is_a?(Numeric)
+
+          span_op.set_metric(Metadata::Ext::Analytics::TAG_SAMPLE_RATE, sample_rate)
+        end
+
+        def set_measured(span_op, value = true)
+          return if span_op.nil?
+
+          value = value == true || value == 1 ? 1 : 0 # rubocop:disable Style/MultipleComparison
+          span_op.set_metric(Metadata::Ext::Analytics::TAG_MEASURED, value)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/buffer.rb

@@ -0,0 +1,125 @@
+# typed: true
+require 'datadog/core'
+require 'datadog/core/environment/ext'
+require 'datadog/core/buffer/thread_safe'
+require 'datadog/core/buffer/cruby'
+require 'datadog/core/diagnostics/health'
+
+module Datadog
+  module Tracing
+    # Health metrics for trace buffers.
+    module MeasuredBuffer
+      include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
+
+      def initialize(*_)
+        super
+
+        @buffer_accepted = 0
+        @buffer_accepted_lengths = 0
+        @buffer_dropped = 0
+        @buffer_spans = 0
+      end
+
+      def add!(trace)
+        super
+
+        # Emit health metrics
+        measure_accept(trace)
+      end
+
+      def add_all!(traces)
+        super
+
+        # Emit health metrics
+        traces.each { |trace| measure_accept(trace) }
+      end
+
+      def replace!(trace)
+        discarded_trace = super
+
+        # Emit health metrics
+        measure_accept(trace)
+        measure_drop(discarded_trace) if discarded_trace
+
+        discarded_trace
+      end
+
+      # Stored traces are returned and the local buffer is reset.
+      def drain!
+        traces = super
+        measure_pop(traces)
+        traces
+      end
+
+      def measure_accept(trace)
+        @buffer_accepted += 1
+        @buffer_accepted_lengths += trace.length
+
+        @buffer_spans += trace.length
+      rescue StandardError => e
+        Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{Array(e.backtrace).first}")
+      end
+
+      def measure_drop(trace)
+        @buffer_dropped += 1
+
+        @buffer_spans -= trace.length
+      rescue StandardError => e
+        Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{Array(e.backtrace).first}")
+      end
+
+      def measure_pop(traces)
+        # Accepted, cumulative totals
+        Datadog.health_metrics.queue_accepted(@buffer_accepted)
+        Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+
+        # Dropped, cumulative totals
+        Datadog.health_metrics.queue_dropped(@buffer_dropped)
+        # TODO: are we missing a +queue_dropped_lengths+ metric?
+
+        # Queue gauges, current values
+        Datadog.health_metrics.queue_max_length(@max_size)
+        Datadog.health_metrics.queue_spans(@buffer_spans)
+        Datadog.health_metrics.queue_length(traces.length)
+
+        # Reset aggregated metrics
+        @buffer_accepted = 0
+        @buffer_accepted_lengths = 0
+        @buffer_dropped = 0
+        @buffer_spans = 0
+      rescue StandardError => e
+        Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{Array(e.backtrace).first}")
+      end
+    end
+
+    # Trace buffer that stores application traces, has a maximum size, and
+    # can be safely used concurrently on any environment.
+    #
+    # @see Datadog::Core::Buffer::ThreadSafe
+    class ThreadSafeTraceBuffer < Core::Buffer::ThreadSafe
+      prepend MeasuredBuffer
+    end
+
+    # Trace buffer that stores application traces, has a maximum size, and
+    # can be safely used concurrently with CRuby.
+    #
+    # @see Datadog::Core::Buffer::CRuby
+    class CRubyTraceBuffer < Core::Buffer::CRuby
+      prepend MeasuredBuffer
+    end
+
+    # Trace buffer that stores application traces. The buffer has a maximum size and when
+    # the buffer is full, a random trace is discarded. This class is thread-safe and is used
+    # automatically by the ``Tracer`` instance when a ``Span`` is finished.
+    #
+    # We choose the default TraceBuffer implementation for current platform dynamically here.
+    #
+    # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
+    # TODO Making such a change is potentially breaking for users manually configuring the tracer.
+    TraceBuffer = if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
+                    CRubyTraceBuffer
+                  else
+                    ThreadSafeTraceBuffer
+                  end
+  end
+end