fair-ddtrace 0.8.2.a

data/lib/ddtrace/transport.rb

@@ -0,0 +1,162 @@
+require 'thread'
+require 'net/http'
+
+require 'ddtrace/encoding'
+require 'ddtrace/version'
+
+module Datadog
+  # Transport class that handles the spans delivery to the
+  # local trace-agent. The class wraps a Net:HTTP instance
+  # so that the Transport is thread-safe.
+  class HTTPTransport
+    attr_accessor :hostname, :port
+    attr_reader :traces_endpoint, :services_endpoint
+
+    # seconds before the transport timeout
+    TIMEOUT = 1
+
+    # header containing the number of traces in a payload
+    TRACE_COUNT_HEADER = 'X-Datadog-Trace-Count'.freeze
+    RUBY_INTERPRETER = RUBY_VERSION > '1.9' ? RUBY_ENGINE + '-' + RUBY_PLATFORM : 'ruby-' + RUBY_PLATFORM
+
+    def initialize(hostname, port, options = {})
+      @hostname = hostname
+      @port = port
+      @traces_endpoint = '/v0.3/traces'.freeze
+      @services_endpoint = '/v0.3/services'.freeze
+      @compatibility_mode = false
+      @encoder = options.fetch(:encoder, Datadog::Encoding::MsgpackEncoder.new())
+
+      # overwrite the Content-type with the one chosen in the Encoder
+      @headers = options.fetch(:headers, {})
+      @headers['Content-Type'] = @encoder.content_type
+      @headers['Datadog-Meta-Lang'] = 'ruby'
+      @headers['Datadog-Meta-Lang-Version'] = RUBY_VERSION
+      @headers['Datadog-Meta-Lang-Interpreter'] = RUBY_INTERPRETER
+      @headers['Datadog-Meta-Tracer-Version'] = Datadog::VERSION::STRING
+
+      # stats
+      @mutex = Mutex.new
+      @count_success = 0
+      @count_client_error = 0
+      @count_server_error = 0
+      @count_internal_error = 0
+    end
+
+    # route the send to the right endpoint
+    def send(endpoint, data)
+      case endpoint
+      when :services
+        payload = @encoder.encode_services(data)
+        status_code = post(@services_endpoint, payload)
+      when :traces
+        count = data.length
+        payload = @encoder.encode_traces(data)
+        status_code = post(@traces_endpoint, payload, count)
+      else
+        Datadog::Tracer.log.error("Unsupported endpoint: #{endpoint}")
+        return nil
+      end
+
+      return status_code unless downgrade?(status_code) && !@compatibility_mode
+
+      # the API endpoint is not available so we should downgrade the connection and re-try the call
+      downgrade!
+      send(endpoint, data)
+    end
+
+    # send data to the trace-agent; the method is thread-safe
+    def post(url, data, count = nil)
+      Datadog::Tracer.log.debug("Sending data from process: #{Process.pid}")
+      headers = count.nil? ? {} : { TRACE_COUNT_HEADER => count.to_s }
+      headers = headers.merge(@headers)
+      request = Net::HTTP::Post.new(url, headers)
+      request.body = data
+
+      response = Net::HTTP.start(@hostname, @port, read_timeout: TIMEOUT) { |http| http.request(request) }
+      handle_response(response)
+    rescue StandardError => e
+      Datadog::Tracer.log.error(e.message)
+      500
+    end
+
+    # Downgrade the connection to a compatibility version of the HTTPTransport;
+    # this method should target a stable API that works whatever is the agent
+    # or the tracing client versions.
+    def downgrade!
+      @compatibility_mode = true
+      @traces_endpoint = '/v0.2/traces'.freeze
+      @services_endpoint = '/v0.2/services'.freeze
+      @encoder = Datadog::Encoding::JSONEncoder.new()
+      @headers['Content-Type'] = @encoder.content_type
+    end
+
+    def informational?(code)
+      code.between?(100, 199)
+    end
+
+    def success?(code)
+      code.between?(200, 299)
+    end
+
+    def redirect?(code)
+      code.between?(300, 399)
+    end
+
+    def client_error?(code)
+      code.between?(400, 499)
+    end
+
+    def server_error?(code)
+      code.between?(500, 599)
+    end
+
+    # receiving a 404 means that we're targeting an endpoint that is not available
+    # in the trace agent. Usually this means that we've an up-to-date tracing client,
+    # while running an obsolete agent.
+    # receiving a 415 means that we're using an unsupported content-type with an existing
+    # endpoint. Usually this means that we're using a newer encoder with a previous
+    # endpoint. In both cases, we're going to downgrade the transporter encoder so that
+    # it will target a stable API.
+    def downgrade?(code)
+      code == 404 || code == 415
+    end
+
+    # handles the server response; here you can log the trace-agent response
+    # or do something more complex to recover from a possible error. This
+    # function is handled within the HTTP mutex.synchronize so it's thread-safe.
+    def handle_response(response)
+      status_code = response.code.to_i
+
+      if success?(status_code)
+        Datadog::Tracer.log.debug('Payload correctly sent to the trace agent.')
+        @mutex.synchronize { @count_success += 1 }
+      elsif downgrade?(status_code)
+        Datadog::Tracer.log.debug("calling the endpoint but received #{status_code}; downgrading the API")
+      elsif client_error?(status_code)
+        Datadog::Tracer.log.error("Client error: #{response.message}")
+        @mutex.synchronize { @count_client_error += 1 }
+      elsif server_error?(status_code)
+        Datadog::Tracer.log.error("Server error: #{response.message}")
+        @mutex.synchronize { @count_server_error += 1 }
+      end
+
+      status_code
+    rescue StandardError => e
+      Datadog::Tracer.log.error(e.message)
+      @mutex.synchronize { @count_internal_error += 1 }
+      500
+    end
+
+    def stats
+      @mutex.synchronize do
+        {
+          success: @count_success,
+          client_error: @count_client_error,
+          server_error: @count_server_error,
+          internal_error: @count_internal_error
+        }
+      end
+    end
+  end
+end

data/lib/ddtrace/utils.rb

@@ -0,0 +1,16 @@
+require 'thread'
+
+module Datadog
+  # Utils contains low-level utilities, typically to provide pseudo-random trace IDs.
+  module Utils
+    # We use a custom random number generator because we want no interference
+    # with the default one. Using the default prng, we could break code that
+    # would rely on srand/rand sequences.
+    @rnd = Random.new
+
+    # Return a span id
+    def self.next_id
+      @rnd.rand(Datadog::Span::MAX_ID)
+    end
+  end
+end

data/lib/ddtrace/version.rb

@@ -0,0 +1,9 @@
+module Datadog
+  module VERSION
+    MAJOR = 0
+    MINOR = 8
+    PATCH = '2-a'
+
+    STRING = [MAJOR, MINOR, PATCH].compact.join('.')
+  end
+end

data/lib/ddtrace/workers.rb

@@ -0,0 +1,108 @@
+require 'time'
+
+require 'ddtrace/buffer'
+
+module Datadog
+  module Workers
+    # Asynchronous worker that executes a +Send()+ operation after given
+    # seconds. Under the hood, it uses +Concurrent::TimerTask+ so that the thread
+    # will perform a task at regular intervals. The thread can be stopped
+    # with the +stop()+ method and can start with the +start()+ method.
+    class AsyncTransport
+      def initialize(span_interval, service_interval, transport, buff_size, trace_task, service_task)
+        @trace_task = trace_task
+        @service_task = service_task
+        @span_interval = span_interval
+        @service_interval = service_interval
+        @trace_buffer = TraceBuffer.new(buff_size)
+        @service_buffer = TraceBuffer.new(buff_size)
+        @transport = transport
+
+        @worker = nil
+        @run = false
+      end
+
+      # Callback function that process traces and executes the +send_traces()+ method.
+      def callback_traces
+        return if @trace_buffer.empty?
+
+        begin
+          traces = @trace_buffer.pop()
+          @trace_task.call(traces, @transport)
+        rescue StandardError => e
+          # ensures that the thread will not die because of an exception.
+          # TODO[manu]: findout the reason and reschedule the send if it's not
+          # a fatal exception
+          Datadog::Tracer.log.error("Error during traces flush: dropped #{traces.length} items. Cause: #{e}")
+        end
+      end
+
+      # Callback function that process traces and executes the +send_services()+ method.
+      def callback_services
+        return if @service_buffer.empty?
+
+        begin
+          services = @service_buffer.pop()
+          # pick up the latest services hash (this is a FIFO list)
+          # that is different from what we sent before.
+          different = services.inject(false) { |acc, elem| elem != @last_flushed_services ? elem : acc }
+          if different
+            if @service_task.call(different, @transport)
+              @last_flushed_services = different.clone
+            end
+          else
+            Datadog::Tracer.log.debug('No new different services, skipping flush.')
+          end
+        rescue StandardError => e
+          # ensures that the thread will not die because of an exception.
+          # TODO[manu]: findout the reason and reschedule the send if it's not
+          # a fatal exception
+          Datadog::Tracer.log.error("Error during services flush: dropped #{services.length} items. Cause: #{e}")
+        end
+      end
+
+      # Start the timer execution.
+      def start
+        return if @run
+        @run = true
+        @worker = Thread.new() do
+          Datadog::Tracer.log.debug("Starting thread in the process: #{Process.pid}")
+          @last_flushed_services = nil
+          next_send_services = Time.now
+
+          # this loop assumes spans are flushed more often than services
+          while @run
+            callback_traces
+            if Time.now >= next_send_services
+              next_send_services = Time.now + @service_interval
+              callback_services
+            end
+            sleep(@span_interval)
+          end
+        end
+      end
+
+      # Stop the timer execution. Tasks already in the queue will be executed.
+      def stop
+        @run = false
+      end
+
+      # Block until executor shutdown is complete or until timeout seconds have passed.
+      def join
+        @worker.join(10)
+      end
+
+      # Enqueue an item in the trace internal buffer. This operation is thread-safe
+      # because uses the +TraceBuffer+ data structure.
+      def enqueue_trace(trace)
+        @trace_buffer.push(trace)
+      end
+
+      # Enqueue an item in the service internal buffer. This operation is thread-safe.
+      def enqueue_service(service)
+        return if service == {} # no use to send this, not worth it
+        @service_buffer.push(service)
+      end
+    end
+  end
+end

data/lib/ddtrace/writer.rb

@@ -0,0 +1,118 @@
+require 'ddtrace/transport'
+require 'ddtrace/encoding'
+require 'ddtrace/workers'
+
+module Datadog
+  # Traces and services writer that periodically sends data to the trace-agent
+  class Writer
+    attr_reader :transport
+
+    HOSTNAME = 'localhost'.freeze
+    PORT = '8126'.freeze
+
+    def initialize(options = {})
+      # writer and transport parameters
+      @buff_size = options.fetch(:buffer_size, 100)
+      @span_interval = options.fetch(:spans_interval, 1)
+      @service_interval = options.fetch(:services_interval, 120)
+
+      # transport and buffers
+      @transport = options.fetch(:transport, Datadog::HTTPTransport.new(HOSTNAME, PORT))
+      @services = {}
+
+      # handles the thread creation after an eventual fork
+      @mutex_after_fork = Mutex.new
+      @pid = nil
+
+      @traces_flushed = 0
+      @services_flushed = 0
+
+      # one worker for both services and traces, each have their own queues
+      @worker = nil
+    end
+
+    # spawns two different workers for spans and services;
+    # they share the same transport which is thread-safe
+    def start
+      @trace_handler = ->(items, transport) { send_spans(items, transport) }
+      @service_handler = ->(items, transport) { send_services(items, transport) }
+      @worker = Datadog::Workers::AsyncTransport.new(@span_interval,
+                                                     @service_interval,
+                                                     @transport,
+                                                     @buff_size,
+                                                     @trace_handler,
+                                                     @service_handler)
+
+      @worker.start()
+    end
+
+    # stops both workers for spans and services.
+    def stop
+      @worker.stop()
+      @worker = nil
+    end
+
+    # flush spans to the trace-agent, handles spans only
+    def send_spans(traces, transport)
+      return true if traces.empty?
+
+      code = transport.send(:traces, traces)
+
+      if transport.server_error? code # requeue on server error, skip on success or client error
+        traces[0..@buff_size].each do |trace|
+          @worker.enqueue_trace trace
+        end
+        return false
+      end
+
+      @traces_flushed += traces.length()
+      true
+    end
+
+    # flush services to the trace-agent, handles services only
+    def send_services(services, transport)
+      return true if services.empty?
+
+      code = transport.send(:services, services)
+      if transport.server_error? code # requeue on server error, skip on success or client error
+        @worker.enqueue_service services
+        return false
+      end
+
+      @services_flushed += 1
+      true
+    end
+
+    # enqueue the trace for submission to the API
+    def write(trace, services)
+      # In multiprocess environments, the main process initializes the +Writer+ instance and if
+      # the process forks (i.e. a web server like Unicorn or Puma with multiple workers) the new
+      # processes will share the same +Writer+ until the first write (COW). Because of that,
+      # each process owns a different copy of the +@buffer+ after each write and so the
+      # +AsyncTransport+ will not send data to the trace agent.
+      #
+      # This check ensures that if a process doesn't own the current +Writer+, async workers
+      # will be initialized again (but only once for each process).
+      pid = Process.pid
+      @mutex_after_fork.synchronize do
+        if pid != @pid
+          @pid = pid
+          # we should start threads because the worker doesn't own this
+          start()
+        end
+      end
+
+      @worker.enqueue_trace(trace)
+      @worker.enqueue_service(services)
+    end
+
+    # stats returns a dictionary of stats about the writer.
+    def stats
+      {
+        traces_flushed: @traces_flushed,
+        services_flushed: @services_flushed,
+        transport: @transport.stats
+      }
+    end
+  end
+end

metadata

@@ -0,0 +1,208 @@
+--- !ruby/object:Gem::Specification
+name: fair-ddtrace
+version: !ruby/object:Gem::Version
+  version: 0.8.2.a
+platform: ruby
+authors:
+- Fair
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-10-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.47'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.47'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.10.1
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: |
+  ddtrace is Datadog’s tracing client for Ruby. It is used to trace requests
+  as they flow across web servers, databases and microservices so that developers
+  have great visiblity into bottlenecks and troublesome requests.
+  Find the original version at: https://github.com/DataDog/dd-trace-rb
+email:
+- catherinec@fair.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env"
+- ".gitignore"
+- ".rubocop.yml"
+- ".yardopts"
+- Appraisals
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- circle.yml
+- ddtrace.gemspec
+- docker-compose.yml
+- docs/GettingStarted.md
+- gemfiles/contrib.gemfile
+- gemfiles/contrib_old.gemfile
+- gemfiles/rails30_postgres.gemfile
+- gemfiles/rails30_postgres_sidekiq.gemfile
+- gemfiles/rails32_mysql2.gemfile
+- gemfiles/rails32_postgres.gemfile
+- gemfiles/rails32_postgres_redis.gemfile
+- gemfiles/rails32_postgres_sidekiq.gemfile
+- gemfiles/rails4_mysql2.gemfile
+- gemfiles/rails4_postgres.gemfile
+- gemfiles/rails4_postgres_redis.gemfile
+- gemfiles/rails4_postgres_sidekiq.gemfile
+- gemfiles/rails5_mysql2.gemfile
+- gemfiles/rails5_postgres.gemfile
+- gemfiles/rails5_postgres_redis.gemfile
+- gemfiles/rails5_postgres_sidekiq.gemfile
+- lib/ddtrace.rb
+- lib/ddtrace/buffer.rb
+- lib/ddtrace/context.rb
+- lib/ddtrace/contrib/active_record/patcher.rb
+- lib/ddtrace/contrib/elasticsearch/patcher.rb
+- lib/ddtrace/contrib/elasticsearch/quantize.rb
+- lib/ddtrace/contrib/grape/endpoint.rb
+- lib/ddtrace/contrib/grape/patcher.rb
+- lib/ddtrace/contrib/http/patcher.rb
+- lib/ddtrace/contrib/rack/middlewares.rb
+- lib/ddtrace/contrib/rails/action_controller.rb
+- lib/ddtrace/contrib/rails/action_view.rb
+- lib/ddtrace/contrib/rails/active_record.rb
+- lib/ddtrace/contrib/rails/active_support.rb
+- lib/ddtrace/contrib/rails/core_extensions.rb
+- lib/ddtrace/contrib/rails/framework.rb
+- lib/ddtrace/contrib/rails/middlewares.rb
+- lib/ddtrace/contrib/rails/utils.rb
+- lib/ddtrace/contrib/redis/patcher.rb
+- lib/ddtrace/contrib/redis/quantize.rb
+- lib/ddtrace/contrib/redis/tags.rb
+- lib/ddtrace/contrib/sidekiq/tracer.rb
+- lib/ddtrace/contrib/sinatra/tracer.rb
+- lib/ddtrace/distributed.rb
+- lib/ddtrace/encoding.rb
+- lib/ddtrace/error.rb
+- lib/ddtrace/ext/app_types.rb
+- lib/ddtrace/ext/cache.rb
+- lib/ddtrace/ext/distributed.rb
+- lib/ddtrace/ext/errors.rb
+- lib/ddtrace/ext/http.rb
+- lib/ddtrace/ext/net.rb
+- lib/ddtrace/ext/redis.rb
+- lib/ddtrace/ext/sql.rb
+- lib/ddtrace/logger.rb
+- lib/ddtrace/monkey.rb
+- lib/ddtrace/pin.rb
+- lib/ddtrace/provider.rb
+- lib/ddtrace/sampler.rb
+- lib/ddtrace/span.rb
+- lib/ddtrace/tracer.rb
+- lib/ddtrace/transport.rb
+- lib/ddtrace/utils.rb
+- lib/ddtrace/version.rb
+- lib/ddtrace/workers.rb
+- lib/ddtrace/writer.rb
+homepage: https://github.com/wearefair/dd-trace-rb
+licenses:
+- BSD-3-Clause
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: Fair fork of Datadog tracing code for your Ruby applications
+test_files: []