ddtrace 0.54.1 → 1.0.0.beta2

data/lib/datadog/tracing/tracer.rb

@@ -0,0 +1,505 @@
+# typed: true
+
+require 'datadog/core'
+require 'datadog/core/environment/ext'
+require 'datadog/core/environment/socket'
+
+require 'datadog/tracing/correlation'
+require 'datadog/tracing/event'
+require 'datadog/tracing/flush'
+require 'datadog/tracing/context_provider'
+require 'datadog/tracing/sampling/all_sampler'
+require 'datadog/tracing/sampling/rule_sampler'
+require 'datadog/tracing/sampling/priority_sampler'
+require 'datadog/tracing/span_operation'
+require 'datadog/tracing/trace_digest'
+require 'datadog/tracing/trace_operation'
+require 'datadog/tracing/writer'
+
+module Datadog
+  module Tracing
+    # A {Datadog::Tracing::Tracer} keeps track of the time spent by an application processing a single operation. For
+    # example, a trace can be used to track the entire time spent processing a complicated web request.
+    # Even though the request may require multiple resources and machines to handle the request, all
+    # of these function calls and sub-requests would be encapsulated within a single trace.
+    # rubocop:disable Metrics/ClassLength
+    class Tracer
+      attr_reader \
+        :trace_flush,
+        :provider,
+        :sampler,
+        :tags
+
+      attr_accessor \
+        :default_service,
+        :enabled,
+        :writer
+
+      # Initialize a new {Datadog::Tracing::Tracer} used to create, sample and submit spans that measure the
+      # time of sections of code.
+      #
+      # @param trace_flush [Datadog::Tracing::TraceFlush] responsible for flushing spans from the execution context
+      # @param context_provider [Datadog::Tracing::DefaultContextProvider] ensures different
+      #                         execution contexts have distinct traces
+      # @param default_service [String] A fallback value for {Datadog::Tracing::Span#service}, as spans without
+      #                        service are rejected
+      # @param enabled [Boolean] set if the tracer submits or not spans to the local agent
+      # @param sampler [Datadog::Tracing::Sampler] a tracer sampler, responsible for filtering out spans when needed
+      # @param tags [Hash] default tags added to all spans
+      # @param writer [Datadog::Tracing::Writer] consumes traces returned by the provided +trace_flush+
+      def initialize(
+        trace_flush: Flush::Finished.new,
+        context_provider: DefaultContextProvider.new,
+        default_service: Core::Environment::Ext::FALLBACK_SERVICE_NAME,
+        enabled: true,
+        sampler: Sampling::PrioritySampler.new(
+          base_sampler: Sampling::AllSampler.new,
+          post_sampler: Sampling::RuleSampler.new
+        ),
+        tags: {},
+        writer: Writer.new
+      )
+        @trace_flush = trace_flush
+        @default_service = default_service
+        @enabled = enabled
+        @provider = context_provider
+        @sampler = sampler
+        @tags = tags
+        @writer = writer
+      end
+
+      # Return a {Datadog::Tracing::SpanOperation span_op} and {Datadog::Tracing::TraceOperation trace_op}
+      # that will trace an operation called `name`.
+      #
+      # You could trace your code using a <tt>do-block</tt> like:
+      #
+      # ```
+      # tracer.trace('web.request') do |span_op, trace_op|
+      #   span_op.service = 'my-web-site'
+      #   span_op.resource = '/'
+      #   span_op.set_tag('http.method', request.request_method)
+      #   do_something()
+      # end
+      # ```
+      #
+      # The {#trace} method can also be used without a block in this way:
+      # ```
+      # span_op = tracer.trace('web.request', service: 'my-web-site')
+      # do_something()
+      # span_op.finish()
+      # ```
+      #
+      # Remember that in this case, calling {Datadog::Tracing::SpanOperation#finish} is mandatory.
+      #
+      # When a Trace is started, {#trace} will store the created span; subsequent spans will
+      # become its children and will inherit some properties:
+      # ```
+      # parent = tracer.trace('parent')   # has no parent span
+      # child  = tracer.trace('child')    # is a child of 'parent'
+      # child.finish()
+      # parent.finish()
+      # parent2 = tracer.trace('parent2') # has no parent span
+      # parent2.finish()
+      # ```
+      #
+      # @param [String] name {Datadog::Tracing::Span} operation name.
+      #   See {https://docs.datadoghq.com/tracing/guide/configuring-primary-operation/ Primary Operations in Services}.
+      # @param [Datadog::Tracing::TraceDigest] continue_from continue a trace from a {Datadog::Tracing::TraceDigest}.
+      #   Used for linking traces that are executed asynchronously.
+      # @param [Proc] on_error a block that overrides error handling behavior for this operation.
+      # @param [String] resource the resource this span refers, or `name` if it's missing
+      # @param [String] service the service name for this span.
+      # @param [Time] start_time time which the span should have started.
+      # @param [Hash<String,String>] tags extra tags which should be added to the span.
+      # @param [String] type the type of the span. See {Datadog::Tracing::Metadata::Ext::AppTypes}.
+      # @return [Object] If a block is provided, returns the result of the block execution.
+      # @return [Datadog::Tracing::SpanOperation] If no block is provided, returns the active,
+      #         unfinished {Datadog::Tracing::SpanOperation}.
+      # @yield Optional block where new newly created {Datadog::Tracing::SpanOperation} captures the execution.
+      # @yieldparam [Datadog::Tracing::SpanOperation] span_op the newly created and active [Datadog::Tracing::SpanOperation]
+      # @yieldparam [Datadog::Tracing::TraceOperation] trace_op the active [Datadog::Tracing::TraceOperation]
+      # rubocop:disable Lint/UnderscorePrefixedVariableName
+      # rubocop:disable Metrics/MethodLength
+      def trace(
+        name,
+        continue_from: nil,
+        on_error: nil,
+        resource: nil,
+        service: nil,
+        start_time: nil,
+        tags: nil,
+        type: nil,
+        span_type: nil,
+        _context: nil,
+        &block
+      )
+        return skip_trace(name, &block) unless enabled
+
+        context, trace = nil
+
+        # Resolve the trace
+        begin
+          context = _context || call_context
+          active_trace = context.active_trace
+          trace = if continue_from || active_trace.nil?
+                    start_trace(continue_from: continue_from)
+                  else
+                    active_trace
+                  end
+        rescue StandardError => e
+          Datadog.logger.debug { "Failed to trace: #{e}" }
+
+          # Tracing failed: fallback and run code without tracing.
+          return skip_trace(name, &block)
+        end
+
+        # Activate and start the trace
+        if block
+          context.activate!(trace) do
+            start_span(
+              name,
+              on_error: on_error,
+              resource: resource,
+              service: service,
+              start_time: start_time,
+              tags: tags,
+              type: span_type || type,
+              _trace: trace,
+              &block
+            )
+          end
+        else
+          # Setup trace activation/deactivation
+          manual_trace_activation!(context, trace)
+
+          # Return the new span
+          start_span(
+            name,
+            on_error: on_error,
+            resource: resource,
+            service: service,
+            start_time: start_time,
+            tags: tags,
+            type: span_type || type,
+            _trace: trace
+          )
+        end
+      end
+      # rubocop:enable Lint/UnderscorePrefixedVariableName
+      # rubocop:enable Metrics/MethodLength
+
+      # Set the given key / value tag pair at the tracer level. These tags will be
+      # appended to each span created by the tracer. Keys and values must be strings.
+      # @example
+      #   tracer.set_tags('env' => 'prod', 'component' => 'core')
+      def set_tags(tags)
+        string_tags = tags.collect { |k, v| [k.to_s, v] }.to_h
+        @tags = @tags.merge(string_tags)
+      end
+
+      # The active, unfinished trace, representing the current instrumentation context.
+      #
+      # The active trace is thread-local.
+      #
+      # @param [Thread] key Thread to retrieve trace from. Defaults to current thread. For internal use only.
+      # @return [Datadog::Tracing::TraceSegment] the active trace
+      # @return [nil] if no trace is active
+      def active_trace(key = nil)
+        call_context(key).active_trace
+      end
+
+      # The active, unfinished span, representing the currently instrumented application section.
+      #
+      # The active span belongs to an {.active_trace}.
+      #
+      # @param [Thread] key Thread to retrieve trace from. Defaults to current thread. For internal use only.
+      # @return [Datadog::Tracing::SpanOperation] the active span
+      # @return [nil] if no trace is active, and thus no span is active
+      def active_span(key = nil)
+        trace = active_trace(key)
+        trace.active_span if trace
+      end
+
+      # Information about the currently active trace.
+      #
+      # The most common use cases are tagging log messages and metrics.
+      #
+      # @param [Thread] key Thread to retrieve trace from. Defaults to current thread. For internal use only.
+      # @return [Datadog::Tracing::Correlation::Identifier] correlation object
+      def active_correlation(key = nil)
+        trace = active_trace(key)
+        Correlation.identifier_from_digest(
+          trace && trace.to_digest
+        )
+      end
+
+      # Setup a new trace to continue from where another
+      # trace left off.
+      #
+      # Used to continue distributed or async traces.
+      #
+      # @param [Datadog::Tracing::TraceDigest] digest continue from the {Datadog::Tracing::TraceDigest}.
+      # @param [Thread] key Thread to retrieve trace from. Defaults to current thread. For internal use only.
+      # @return [Object] If a block is provided, the result of the block execution.
+      # @return [Datadog::Tracing::TraceOperation] If no block, the active {Datadog::Tracing::TraceOperation}.
+      # @yield Optional block where this {#continue_trace!} `digest` scope is active.
+      #   If no block, the `digest` remains active after {#continue_trace!} returns.
+      def continue_trace!(digest, key = nil, &block)
+        # Only accept {TraceDigest} as a digest.
+        # Otherwise, create a new execution context.
+        digest = nil unless digest.is_a?(TraceDigest)
+
+        # Start a new trace from the digest
+        context = call_context(key)
+        original_trace = active_trace(key)
+        trace = start_trace(continue_from: digest)
+
+        # If block hasn't been given; we need to manually deactivate
+        # this trace. Subscribe to the trace finished event to do this.
+        subscribe_trace_deactivation!(context, trace, original_trace) unless block
+
+        context.activate!(trace, &block)
+      end
+
+      # @!visibility private
+      # TODO: make this private
+      def trace_completed
+        @trace_completed ||= TraceCompleted.new
+      end
+
+      # Triggered whenever a trace is completed
+      class TraceCompleted < Tracing::Event
+        def initialize
+          super(:trace_completed)
+        end
+
+        # NOTE: Ignore Rubocop rule. This definition allows for
+        #       description of and constraints on arguments.
+        # rubocop:disable Lint/UselessMethodDefinition
+        def publish(trace)
+          super(trace)
+        end
+        # rubocop:enable Lint/UselessMethodDefinition
+      end
+
+      # Shorthand that calls the `shutdown!` method of a registered worker.
+      # It's useful to ensure that the Trace Buffer is properly flushed before
+      # shutting down the application.
+      #
+      # @example
+      #   tracer.trace('operation_name', service='rake_tasks') do |span_op|
+      #     span_op.set_tag('task.name', 'script')
+      #   end
+      #
+      #   tracer.shutdown!
+      def shutdown!
+        return unless @enabled
+
+        @writer.stop if @writer
+      end
+
+      private
+
+      # Return the current active {Context} for this traced execution. This method is
+      # automatically called when calling Tracer.trace or Tracer.start_span,
+      # but it can be used in the application code during manual instrumentation.
+      #
+      # This method makes use of a {ContextProvider} that is automatically set during the tracer
+      # initialization, or while using a library instrumentation.
+      #
+      # @param [Thread] key Thread to retrieve tracer from. Defaults to current thread.
+      def call_context(key = nil)
+        @provider.context(key)
+      end
+
+      def build_trace(digest = nil)
+        # Resolve hostname if configured
+        hostname = Core::Environment::Socket.hostname if Datadog.configuration.tracing.report_hostname
+        hostname = hostname && !hostname.empty? ? hostname : nil
+
+        if digest
+          TraceOperation.new(
+            hostname: hostname,
+            id: digest.trace_id,
+            origin: digest.trace_origin,
+            parent_span_id: digest.span_id,
+            sampling_priority: digest.trace_sampling_priority
+          )
+        else
+          TraceOperation.new(
+            hostname: hostname
+          )
+        end
+      end
+
+      def bind_trace_events!(trace_op)
+        events = trace_op.send(:events)
+
+        events.span_before_start.subscribe do |event_span_op, event_trace_op|
+          event_trace_op.service ||= @default_service
+          event_span_op.service ||= @default_service
+          sample_trace(event_trace_op) if event_span_op && event_span_op.parent_id == 0
+        end
+
+        events.span_finished.subscribe do |_event_span, event_trace_op|
+          flush_trace(event_trace_op)
+        end
+      end
+
+      def start_trace(continue_from: nil)
+        # Build a new trace using digest if provided.
+        trace = build_trace(continue_from)
+
+        # Bind trace events: sample trace, set default service, flush spans.
+        bind_trace_events!(trace)
+
+        trace
+      end
+
+      # rubocop:disable Lint/UnderscorePrefixedVariableName
+      def start_span(
+        name,
+        continue_from: nil,
+        on_error: nil,
+        resource: nil,
+        service: nil,
+        start_time: nil,
+        tags: nil,
+        type: nil,
+        _trace: nil,
+        &block
+      )
+        trace = _trace || start_trace(continue_from: continue_from)
+
+        if block
+          # Ignore start time if a block has been given
+          trace.measure(
+            name,
+            events: build_span_events,
+            on_error: on_error,
+            resource: resource,
+            service: service,
+            tags: resolve_tags(tags),
+            type: type,
+            &block
+          )
+        else
+          # Return the new span
+          span = trace.build_span(
+            name,
+            events: build_span_events,
+            on_error: on_error,
+            resource: resource,
+            service: service,
+            start_time: start_time,
+            tags: resolve_tags(tags),
+            type: type
+          )
+
+          span.start(start_time)
+          span
+        end
+      end
+      # rubocop:enable Lint/UnderscorePrefixedVariableName
+
+      def build_span_events(events = nil)
+        case events
+        when SpanOperation::Events
+          events
+        when Hash
+          SpanOperation::Events.build(events)
+        else
+          SpanOperation::Events.new
+        end
+      end
+
+      def resolve_tags(tags)
+        if @tags.any? && tags
+          # Combine default tags with provided tags,
+          # preferring provided tags.
+          @tags.merge(tags)
+        else
+          # Use provided tags or default tags if none.
+          tags || @tags.dup
+        end
+      end
+
+      # Manually activate and deactivate the trace, when the span completes.
+      def manual_trace_activation!(context, trace)
+        # Get the original trace to restore
+        original_trace = context.active_trace
+
+        # Setup the deactivation callback
+        subscribe_trace_deactivation!(context, trace, original_trace)
+
+        # Activate the trace
+        # Skip this, if it would have no effect.
+        context.activate!(trace) unless trace == original_trace
+      end
+
+      # Reactivate the original trace when trace completes
+      def subscribe_trace_deactivation!(context, trace, original_trace)
+        # Don't override this event if it's set.
+        # The original event should reactivate the original trace correctly.
+        #
+        # This happens when multiple manually-activation spans are created:
+        # ```ruby
+        # tracer.trace('parent') do
+        #   span1 = tracer.trace('first') # Registers trace deactivation back to `parent` span.
+        #   span2 = tracer.trace('second') # Tries to register trace deactivation back to `first`, which is not correct.
+        # end
+        # ```
+        return if trace.send(:events).trace_finished.deactivate_trace_subscribed?
+
+        trace.send(:events).trace_finished.subscribe_deactivate_trace do
+          context.activate!(original_trace)
+        end
+      end
+
+      # Sample a span, tagging the trace as appropriate.
+      def sample_trace(trace_op)
+        begin
+          @sampler.sample!(trace_op)
+        rescue StandardError => e
+          Datadog.logger.debug { "Failed to sample trace: #{e}" }
+        end
+      end
+
+      # Flush finished spans from the trace buffer, send them to writer.
+      def flush_trace(trace_op)
+        begin
+          trace = @trace_flush.consume!(trace_op)
+          write(trace) if trace && !trace.empty?
+        rescue StandardError => e
+          Datadog.logger.debug { "Failed to flush trace: #{e}" }
+        end
+      end
+
+      # Send the trace to the writer to enqueue the spans list in the agent
+      # sending queue.
+      def write(trace)
+        return unless trace && @writer
+
+        if Datadog.configuration.diagnostics.debug
+          Datadog.logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
+        end
+
+        @writer.write(trace)
+        trace_completed.publish(trace)
+      end
+
+      # TODO: Make these dummy objects singletons to preserve memory.
+      def skip_trace(name)
+        span = SpanOperation.new(name)
+
+        if block_given?
+          trace = TraceOperation.new
+          yield(span, trace)
+        else
+          span
+        end
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/datadog/tracing/workers/trace_writer.rb

@@ -0,0 +1,195 @@
+# typed: false
+
+require 'datadog/core'
+require 'datadog/core/worker'
+require 'datadog/core/workers/async'
+require 'datadog/core/workers/polling'
+require 'datadog/core/workers/queue'
+
+require 'datadog/tracing/buffer'
+require 'datadog/tracing/pipeline'
+require 'datadog/tracing/event'
+
+require 'ddtrace/transport/http'
+
+module Datadog
+  module Tracing
+    module Workers
+      # Writes traces to transport synchronously
+      class TraceWriter < Core::Worker
+        attr_reader \
+          :transport
+
+        # rubocop:disable Lint/MissingSuper
+        def initialize(options = {})
+          transport_options = options.fetch(:transport_options, {})
+
+          transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
+
+          @transport = options.fetch(:transport) do
+            Transport::HTTP.default(**transport_options)
+          end
+        end
+        # rubocop:enable Lint/MissingSuper
+
+        def perform(traces)
+          write_traces(traces)
+        end
+
+        def write(trace)
+          write_traces([trace])
+        end
+
+        def write_traces(traces)
+          traces = process_traces(traces)
+          flush_traces(traces)
+        rescue StandardError => e
+          Datadog.logger.error(
+            "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
+          )
+        end
+
+        def process_traces(traces)
+          # Run traces through the processing pipeline
+          Pipeline.process!(traces)
+        end
+
+        def flush_traces(traces)
+          transport.send_traces(traces).tap do |response|
+            flush_completed.publish(response)
+          end
+        end
+
+        # TODO: Register `Datadog::Core::Diagnostics::EnvironmentLogger.log!`
+        # TODO: as a flush_completed subscriber when the `TraceWriter`
+        # TODO: instantiation code is implemented.
+        def flush_completed
+          @flush_completed ||= FlushCompleted.new
+        end
+
+        # Flush completed event for worker
+        class FlushCompleted < Event
+          def initialize
+            super(:flush_completed)
+          end
+        end
+      end
+
+      # Writes traces to transport asynchronously,
+      # using a thread & buffer.
+      class AsyncTraceWriter < TraceWriter
+        include Core::Workers::Queue
+        include Core::Workers::Polling
+
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        FORK_POLICY_ASYNC = :async
+        FORK_POLICY_SYNC = :sync
+
+        attr_writer \
+          :async
+
+        def initialize(options = {})
+          # Workers::TraceWriter settings
+          super
+
+          # Workers::Polling settings
+          self.enabled = options.fetch(:enabled, true)
+
+          # Workers::Async::Thread settings
+          @async = true
+          self.fork_policy = options.fetch(:fork_policy, FORK_POLICY_ASYNC)
+
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = options[:interval] if options.key?(:interval)
+          self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+          self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+          # Workers::Queue settings
+          @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+          self.buffer = TraceBuffer.new(@buffer_size)
+        end
+
+        # NOTE: #perform is wrapped by other modules:
+        #       Polling --> Async --> IntervalLoop --> AsyncTraceWriter --> TraceWriter
+        #
+        # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#perform is spec'd to return the
+        # result from the writer, whereas this method always returns nil.
+        def perform(traces)
+          super(traces).tap do |responses|
+            loop_back_off! if responses.find(&:server_error?)
+          end
+
+          nil
+        end
+
+        def stop(*args)
+          buffer.close if running?
+          super
+        end
+
+        def enqueue(trace)
+          buffer.push(trace)
+        end
+
+        def dequeue
+          # Wrap results in Array because they are
+          # splatted as args against TraceWriter#perform.
+          [buffer.pop]
+        end
+
+        # Are there more traces to be processed next?
+        def work_pending?
+          !buffer.empty?
+        end
+
+        def async?
+          @async == true
+        end
+
+        def fork_policy=(policy)
+          # Translate to Workers::Async::Thread policy
+          thread_fork_policy =  case policy
+                                when Core::Workers::Async::Thread::FORK_POLICY_STOP
+                                  policy
+                                when FORK_POLICY_SYNC
+                                  # Stop the async thread because the writer
+                                  # will bypass and run synchronously.
+                                  Core::Workers::Async::Thread::FORK_POLICY_STOP
+                                else
+                                  Core::Workers::Async::Thread::FORK_POLICY_RESTART
+                                end
+
+          # Update thread fork policy
+          super(thread_fork_policy)
+
+          # Update local policy
+          @writer_fork_policy = policy
+        end
+
+        def after_fork
+          # In multiprocess environments, forks will share the same buffer until its written to.
+          # A.K.A. copy-on-write. We don't want forks to write traces generated from another process.
+          # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+          self.buffer = TraceBuffer.new(@buffer_size)
+
+          # Switch to synchronous mode if configured to do so.
+          # In some cases synchronous writing is preferred because the fork will be short lived.
+          @async = false if @writer_fork_policy == FORK_POLICY_SYNC
+        end
+
+        # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#write is spec'd to return the
+        # result from the writer, whereas this method returns something else when running in async mode.
+        def write(trace)
+          # Start worker thread. If the process has forked, it will trigger #after_fork to
+          # reconfigure the worker accordingly.
+          # NOTE: It's important we do this before queuing or it will drop the current trace,
+          #       because #after_fork resets the buffer.
+          perform
+
+          # Queue the trace if running asynchronously, otherwise short-circuit and write it directly.
+          async? ? enqueue(trace) : write_traces([trace])
+        end
+      end
+    end
+  end
+end