epsagon 0.0.20 → 0.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6064c0bad59a827b3521f5e26dabfad76f4e80ba729be380d96fd682a178fc6e
-  data.tar.gz: d84a5475b4b0c81cb7f20fb9a8baeca13a1f83580affadc5932cc37023e82a6b
+  metadata.gz: e4dd6a90b916b237e688e45551af0324cebf6dc8288c5be00c2c360604748aa7
+  data.tar.gz: e70b552116e3a68515e6363352527ba0e1a438a1b8f26d9882358c61e7f2f5f7
 SHA512:
-  metadata.gz: bbfa836f01e358d43452b1d2eb64c37870f50a6241d5e0748b965eb606ddb5c049e10b32e4467e04a9153227fdcf62ec57e19e2ee0005c3e632c31278fded005
-  data.tar.gz: 1ba143caeb354cf9fc06d567d6f26ae27df6ef9ff01e820baf0ab8b677e992639ec13f56f04fa45a907c5d944d3eb8b9061436c8f819e7431a1240989c7d5afc
+  metadata.gz: '048ca5df1c956861b25634b497ceb8b1e6bcb45ff171a14e4c61bb1d1ea8731b5cf87237a158debe2ff146b81ba72445c1050bab2ba875596624cef82b1feec1'
+  data.tar.gz: 7474c11f77a5f24da6a23e7c825e1f4cf1e2ef0111cabd0ece1c68322a3e5cc8345e499618796f41de6484b2cdfb98c95abc7d1eeca3c13b2abb7a63d00a60b3

data/lib/epsagon.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
-
+require 'json'
 require 'rubygems'
 require 'net/http'
 require 'bundler/setup'
 require 'opentelemetry/sdk'
 require 'opentelemetry/exporter/otlp'
+require 'opentelemetry/instrumentation/sidekiq'
 
 require_relative 'instrumentation/sinatra'
 require_relative 'instrumentation/net_http'
@@ -13,26 +14,29 @@ require_relative 'instrumentation/aws_sdk'
 require_relative 'instrumentation/rails'
 require_relative 'util'
 require_relative 'epsagon_constants'
+require_relative 'exporter_extension'
 
 Bundler.require
 
 # Epsagon tracing main entry point
 module Epsagon
-  
   DEFAULT_BACKEND = 'opentelemetry.tc.epsagon.com:443/traces'
+  DEFAULT_IGNORE_DOMAINS = ['newrelic.com']
 
-  @@epsagon_config = {
-    metadata_only: ENV['EPSAGON_METADATA']&.to_s&.downcase != 'false',
-    debug: ENV['EPSAGON_DEBUG']&.to_s&.downcase == 'true',
-    token: ENV['EPSAGON_TOKEN'],
-    app_name: ENV['EPSAGON_APP_NAME'],
-    max_attribute_size: ENV['EPSAGON_MAX_ATTRIBUTE_SIZE'] || 5000,
-    backend: ENV['EPSAGON_BACKEND'] || DEFAULT_BACKEND
-  }
+  @@epsagon_config = {}
 
   module_function
 
   def init(**args)
+    @@epsagon_config = {
+      metadata_only: ENV['EPSAGON_METADATA']&.to_s&.downcase != 'false',
+      debug: ENV['EPSAGON_DEBUG']&.to_s&.downcase == 'true',
+      token: ENV['EPSAGON_TOKEN'] || '',
+      app_name: ENV['EPSAGON_APP_NAME'] || '',
+      max_attribute_size: ENV['EPSAGON_MAX_ATTRIBUTE_SIZE'] || 5000,
+      backend: ENV['EPSAGON_BACKEND'] || DEFAULT_BACKEND,
+      ignore_domains: ENV['EPSAGON_IGNORE_DOMAINS'] || DEFAULT_IGNORE_DOMAINS
+    }
     @@epsagon_config.merge!(args)
     OpenTelemetry::SDK.configure
   end
@@ -45,9 +49,10 @@ module Epsagon
 
   def epsagon_confs(configurator)
     configurator.resource = OpenTelemetry::SDK::Resources::Resource.telemetry_sdk.merge(
-      OpenTelemetry::SDK::Resources::Resource.create({ 
+      OpenTelemetry::SDK::Resources::Resource.create({
         'application' => @@epsagon_config[:app_name],
-        'epsagon.version' => EpsagonConstants::VERSION
+        'epsagon.version' => EpsagonConstants::VERSION,
+        'epsagon.metadata_only' => @@epsagon_config[:metadata_only]
       })
     )
     configurator.use 'EpsagonSinatraInstrumentation', { epsagon: @@epsagon_config }
@@ -55,8 +60,7 @@ module Epsagon
     configurator.use 'EpsagonFaradayInstrumentation', { epsagon: @@epsagon_config }
     configurator.use 'EpsagonAwsSdkInstrumentation', { epsagon: @@epsagon_config }
     configurator.use 'EpsagonRailsInstrumentation', { epsagon: @@epsagon_config }
-    configurator.use 'OpenTelemetry::Instrumentation::Sidekiq'
-
+    configurator.use 'OpenTelemetry::Instrumentation::Sidekiq', { epsagon: @@epsagon_config }
 
     if @@epsagon_config[:debug]
       configurator.add_span_processor OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(
@@ -97,15 +101,86 @@ module SpanExtension
   def initialize(*args)
     super(*args)
     if @attributes
-      @attributes = Hash[@attributes.map { |k,v|
+      @attributes = Hash[@attributes.select {|k,v| not BLANKS.include? v}.map { |k,v|
         [k, Util.trim_attr(v, Epsagon.get_config[:max_attribute_size])]
       }]
     end
-    
+  end
+end
+
+module SidekiqClientMiddlewareExtension
+
+  def call(_worker_class, job, _queue, _redis_pool)
+    config = OpenTelemetry::Instrumentation::Sidekiq::Instrumentation.instance.config[:epsagon] || {}
+    attributes = {
+      'operation' => job['at'] ? 'perform_at' : 'perform_async',
+      'messaging.system' => 'sidekiq',
+      'messaging.sidekiq.job_class' => job['wrapped']&.to_s || job['class'],
+      'messaging.message_id' => job['jid'],
+      'messaging.destination' => job['queue'],
+      'messaging.destination_kind' => 'queue',
+      'messaging.sidekiq.redis_url' => Sidekiq.options['url'] || Util.redis_default_url
+    }
+    unless config[:metadata_only]
+      attributes.merge!({
+        'messaging.sidekiq.args' => JSON.dump(job['args'])
+      })
+    end
+    tracer.in_span(
+      job['queue'],
+      attributes: attributes,
+      kind: :producer
+    ) do |span|
+      OpenTelemetry.propagation.text.inject(job)
+      span.add_event('created_at', timestamp: job['created_at'])
+      Util.untraced {yield}
+    end
+  end
+end
+
+module SidekiqServerMiddlewareExtension
+  def call(_worker, msg, _queue)
+    config = OpenTelemetry::Instrumentation::Sidekiq::Instrumentation.instance.config[:epsagon] || {}
+    parent_context = OpenTelemetry.propagation.text.extract(msg)
+    attributes = {
+        'operation' => 'perform',
+        'messaging.system' => 'sidekiq',
+        'messaging.sidekiq.job_class' => msg['wrapped']&.to_s || msg['class'],
+        'messaging.message_id' => msg['jid'],
+        'messaging.destination' => msg['queue'],
+        'messaging.destination_kind' => 'queue',
+        'messaging.sidekiq.redis_url' => Sidekiq.options['url'] || Util.redis_default_url
+    }
+    runner_attributes = {
+      'type' => 'sidekiq_worker',
+      'messaging.sidekiq.redis_url' => Sidekiq.options['url'] || Util.redis_default_url,
+
+    }
+    unless config[:metadata_only]
+      attributes.merge!({
+        'messaging.sidekiq.args' => JSON.dump(msg['args'])
+      })
+    end
+    tracer.in_span(
+      msg['queue'],
+      attributes: attributes,
+      with_parent: parent_context,
+      kind: :consumer
+    ) do |trigger_span|
+      trigger_span.add_event('created_at', timestamp: msg['created_at'])
+      trigger_span.add_event('enqueued_at', timestamp: msg['enqueued_at'])
+      tracer.in_span(msg['wrapped']&.to_s || msg['class'],
+        attributes: runner_attributes,
+        kind: :consumer
+      ) do |runner_span|
+        yield
+      end
+    end
   end
 end
 
 # monkey patch to include epsagon confs
+
 module OpenTelemetry
   # monkey patch inner SDK module
   module SDK
@@ -122,4 +197,35 @@ module OpenTelemetry
       end
     end
   end
+  module Instrumentation
+    module Sidekiq
+      class Instrumentation
+        def add_server_middleware
+          ::Sidekiq.configure_server do |config|
+            config.server_middleware do |chain|
+              chain.add Middlewares::Server::TracerMiddleware
+            end
+          end
+
+          if defined?(::Sidekiq::Testing) # rubocop:disable Style/GuardClause
+            ::Sidekiq::Testing.server_middleware do |chain|
+              chain.add Middlewares::Server::TracerMiddleware
+            end
+          end
+        end
+      end
+      module Middlewares
+        module Client
+          class TracerMiddleware
+            prepend SidekiqClientMiddlewareExtension
+          end
+        end
+        module Server
+          class TracerMiddleware
+            prepend SidekiqServerMiddlewareExtension
+          end
+        end
+      end
+    end
+  end
 end

data/lib/epsagon_constants.rb

@@ -1,3 +1,3 @@
 module EpsagonConstants
-	VERSION = '0.0.17'
+	VERSION = '0.0.25'
 end

data/lib/exporter_extension.rb

@@ -0,0 +1,78 @@
+
+
+module OpenTelemetry
+  module Exporter
+    module OTLP
+      # An OpenTelemetry trace exporter that sends spans over HTTP as Protobuf encoded OTLP ExportTraceServiceRequests.
+      class Exporter # rubocop:disable Metrics/ClassLength
+        def send_bytes(bytes, timeout:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+          retry_count = 0
+          timeout ||= @timeout
+          start_time = Time.now
+          untraced do # rubocop:disable Metrics/BlockLength
+            request = Net::HTTP::Post.new(@path)
+            request.body = if @compression == 'gzip'
+                             request.add_field('Content-Encoding', 'gzip')
+                             Zlib.gzip(bytes)
+                           else
+                             bytes
+                           end
+            request.add_field('Content-Type', 'application/x-protobuf')
+            @headers&.each { |key, value| request.add_field(key, value) }
+
+            remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)
+            return TIMEOUT if remaining_timeout.zero?
+
+            @http.open_timeout = remaining_timeout
+            @http.read_timeout = remaining_timeout
+            @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED
+            @http.start unless @http.started?
+            response = measure_request_duration { @http.request(request) }
+
+            case response
+            when Net::HTTPOK
+              response.body # Read and discard body
+              SUCCESS
+            when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests
+              response.body # Read and discard body
+              redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code)
+              FAILURE
+            when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway
+              response.body # Read and discard body
+              redo if backoff?(retry_count: retry_count += 1, reason: response.code)
+              FAILURE
+            when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError
+              # TODO: decode the body as a google.rpc.Status Protobuf-encoded message when https://github.com/open-telemetry/opentelemetry-collector/issues/1357 is fixed.
+              response.body # Read and discard body
+              FAILURE
+            when Net::HTTPRedirection
+              @http.finish
+              handle_redirect(response['location'])
+              redo if backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code)
+            else
+              @http.finish
+              FAILURE
+            end
+          rescue Net::OpenTimeout, Net::ReadTimeout
+            puts "Epsagon: timeout while sending trace" if Epsagon.get_config[:debug]
+            retry if backoff?(retry_count: retry_count += 1, reason: 'timeout')
+            return FAILURE
+          ensure
+            if Epsagon.get_config[:debug] && response && response.code.to_i >= 400
+              puts "Epsagon: Error while sending trace:"
+              puts "#{response.code} #{response.class.name} #{response.message}"
+              puts "Headers: #{response.to_hash.inspect}"
+              puts response.body
+            end
+          end
+        ensure
+          # Reset timeouts to defaults for the next call.
+          @http.open_timeout = @timeout
+          @http.read_timeout = @timeout
+          @http.write_timeout = @timeout if WRITE_TIMEOUT_SUPPORTED
+        end
+      end
+    end
+  end
+end
+

data/lib/instrumentation/aws_sdk_plugin.rb

@@ -4,6 +4,7 @@ require 'aws-sdk-core'
 require 'opentelemetry/common'
 require 'opentelemetry/sdk'
 
+
 def untraced(&block)
   OpenTelemetry::Trace.with_span(OpenTelemetry::Trace::Span.new, &block)
 end
@@ -17,14 +18,120 @@ end
 
 # Generates Spans for all uses of AWS SDK
 class EpsagonAwsHandler < Seahorse::Client::Handler
+  SPAN_KIND = {
+    'ReceiveMessage' => :consumer,
+    'SendMessage' => :producer,
+    'SendMessageBatch' => :producer,
+    'Publish' => :producer,
+  }
+
   def call(context)
-    tracer.in_span('', kind: :client) do |span|
+    span_name = ''
+    span_kind = :client
+    attributes = {
+      'aws.service' => context.client.class.to_s.split('::')[1].downcase,
+      'aws.operation' => context.operation.name
+    }  
+    attributes['aws.region'] = context.client.config.region unless attributes['aws.service'] == 's3'
+
+    span_kind = SPAN_KIND[attributes['aws.operation']] || span_kind
+    if attributes['aws.service'] == 's3'
+      attributes['aws.s3.bucket'] = context.params[:bucket]
+      span_name = attributes['aws.s3.bucket'] if attributes['aws.s3.bucket']
+      attributes['aws.s3.key'] = context.params[:key]
+      attributes['aws.s3.copy_source'] = context.params[:copy_source]
+    elsif attributes['aws.service'] == 'sqs'
+      queue_url = context.params[:queue_url]
+      queue_name = queue_url ? queue_url[queue_url.rindex('/')+1..-1] : context.params[:queue_name]
+      attributes['aws.sqs.max_number_of_messages'] = context.params[:max_number_of_messages]
+      attributes['aws.sqs.wait_time_seconds'] = context.params[:wait_time_seconds]
+      attributes['aws.sqs.visibility_timeout'] = context.params[:visibility_timeout]
+      attributes['aws.sqs.message_id'] = context.params[:message_id]
+      if queue_name
+        attributes['aws.sqs.queue_name'] = queue_name
+        span_name = attributes['aws.sqs.queue_name'] if attributes['aws.sqs.queue_name']
+      end
+      unless config[:epsagon][:metadata_only]
+        if attributes['aws.operation'] == 'SendMessageBatch'
+          messages_attributes = context.params[:entries].map do |m|
+            record = {
+              'message_attributes' => m[:message_attributes].map {|k,v| [k, v.to_h]},
+              'message_body' => m[:message_body],
+            } 
+          end
+          attributes['aws.sqs.record'] = JSON.dump(messages_attributes) if messages_attributes
+        end
+        attributes['aws.sqs.record.message_body'] = context.params[:message_body]
+        attributes['aws.sqs.record.message_attributes'] = JSON.dump(context.params[:message_attributes]) if context.params[:message_attributes]
+      end
+    elsif attributes['aws.service'] == 'sns'
+      topic_arn = context.params[:topic_arn]
+      topic_name = topic_arn ? topic_arn[topic_arn.rindex(':')+1..-1] : context.params[:name] 
+      span_name = attributes['aws.sns.topic_name'] = topic_name if topic_name
+      unless config[:epsagon][:metadata_only]
+        attributes['aws.sns.subject'] = context.params[:subject]
+        attributes['aws.sns.message'] = context.params[:message]
+        attributes['aws.sns.message_attributes'] = JSON.dump(context.params[:message_attributes]) if context.params[:message_attributes]
+      end
+    end
+    tracer.in_span(span_name, kind: span_kind, attributes: attributes) do |span|
       untraced do
-        @handler.call(context).tap do
-          span.set_attribute('aws.service', context.client.class.to_s.split('::')[1].downcase)
-          span.set_attribute('aws.operation', context.operation.name)
-          span.set_attribute('aws.region', context.client.config.region)
+        @handler.call(context).tap do |result|
+          if attributes['aws.service'] == 's3'
+            modified = context.http_response.headers[:'last-modified']
+            reformatted_modified = modified ? 
+                                  Time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')
+                                  .strftime('%Y-%m-%dT%H:%M:%SZ') :
+                                  nil
+            if context.operation.name == 'GetObject'
+              span.set_attribute('aws.s3.content_length', context.http_response.headers[:'content-length']&.to_i)
+            end
+            span.set_attribute('aws.s3.etag', context.http_response.headers[:etag]&.tr('"',''))
+            span.set_attribute('aws.s3.last_modified', reformatted_modified)
+          elsif attributes['aws.service'] == 'sqs'
+            if context.operation.name == 'SendMessage'
+              span.set_attribute('aws.sqs.record.message_id', result.message_id)
+            end
+            if context.operation.name == 'SendMessageBatch'
+              messages_attributes = result.successful.map do |m|
+                record = {'message_id' => m.message_id}
+                unless config[:epsagon][:metadata_only]
+                  context.params[:entries].each do |e|
+                    record.merge!({
+                      'message_attributes' => e[:message_attributes].map {|k,v| [k, v.to_h]},
+                      'message_body' => e[:message_body],
+                    }) if e[:id] == m.id
+                  end
+                end
+                record
+              end
+              span.set_attribute('aws.sqs.record', JSON.dump(messages_attributes)) if messages_attributes
+            end
+            if context.operation.name == 'ReceiveMessage'
+              messages_attributes = result.messages.map do |m|
+                record = {
+                  'message_id' => m.message_id,
+                  'attributes' => {
+                    'sender_id' => m.attributes['SenderId'],
+                    'sent_timestamp' => m.attributes['SentTimestamp'],
+                    'aws_trace_header' => m.attributes['AWSTraceHeader'],
+                  }
+                }
+                unless config[:epsagon][:metadata_only]
+                  record['message_attributes'] = m.message_attributes.map {|k,v| [k, v.to_h]}
+                  record['message_body'] = m.body
+                end 
+                record
+              end
+              span.set_attribute('aws.sqs.record', JSON.dump(messages_attributes)) if messages_attributes
+            end
+          elsif attributes['aws.service'] == 'sns'
+            span.set_attribute('aws.sns.message_id', result.message_id) if context.operation.name == 'Publish'
+          end
           span.set_attribute('http.status_code', context.http_response.status_code)
+          span.status = OpenTelemetry::Trace::Status.http_to_status(
+            context.http_response.status_code
+          )
         end
       end
     end
@@ -33,4 +140,8 @@ class EpsagonAwsHandler < Seahorse::Client::Handler
   def tracer
     EpsagonAwsSdkInstrumentation.instance.tracer()
   end
+
+  def config
+    EpsagonAwsSdkInstrumentation.instance.config
+  end
 end

data/lib/instrumentation/epsagon_faraday_middleware.rb

@@ -37,11 +37,11 @@ class EpsagonFaradayMiddleware < ::Faraday::Middleware
     unless config[:epsagon][:metadata_only]
       attributes.merge!(Util.epsagon_query_attributes(env.url.query))
       attributes.merge!({
-                          'http.request.path_params' => path_params,
-                          'http.request.headers' => env.request_headers.to_json,
-                          'http.request.body' => env.body,
-                          'http.request.headers.User-Agent' => env.request_headers['User-Agent']
-                        })
+        'http.request.path_params' => path_params,
+        'http.request.headers' => env.request_headers.to_json,
+        'http.request.body' => env.body,
+        'http.request.headers.User-Agent' => env.request_headers['User-Agent']
+      })
     end
 
     tracer.in_span(

data/lib/instrumentation/epsagon_rails_middleware.rb

@@ -130,7 +130,7 @@ class EpsagonRackMiddleware
   def request_span_attributes(env:)
     request = Rack::Request.new(env)
     path, path_params = request.path.split(';')
-    request_headers = JSON.generate(Hash[*env.select { |k, _v| k.start_with? 'HTTP_' }
+    request_headers = JSON.generate(Hash[*env.select { |k, _v| k.to_s.start_with? 'HTTP_' }
       .collect { |k, v| [k.sub(/^HTTP_/, ''), v] }
       .collect { |k, v| [k.split('_').collect(&:capitalize).join('-'), v] }
       .sort

data/lib/instrumentation/net_http.rb

@@ -17,6 +17,7 @@ module EpsagonNetHTTPExtension
   def request(req, body = nil, &block)
     # Do not trace recursive call for starting the connection
     return super(req, body, &block) unless started?
+    return super(req, body, &block) if config[:epsagon][:ignore_domains].any? {|d| @address.include? d}
 
     attributes = Hash[OpenTelemetry::Common::HTTP::ClientContext.attributes]
     path_with_params, query = req.path.split('?')
@@ -55,6 +56,7 @@ module EpsagonNetHTTPExtension
 
   def annotate_span_with_response!(span, response)
     return unless response&.code
+    return unless span.respond_to?(:set_attribute)
 
     status_code = response.code.to_i
 

data/lib/util.rb

@@ -30,4 +30,13 @@ module Util
   		value
   	end
   end
+
+  def self.redis_default_url
+    @@redis_default_url ||= "#{Redis::Client::DEFAULTS[:scheme]}://#{Redis::Client::DEFAULTS[:host]}:#{Redis::Client::DEFAULTS[:port]}/#{Redis::Client::DEFAULTS[:db]}"
+  end
+
+  def self.untraced(&block)
+	  OpenTelemetry::Trace.with_span(OpenTelemetry::Trace::Span.new, &block)
+	end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: epsagon
 version: !ruby/object:Gem::Version
-  version: 0.0.20
+  version: 0.0.25
 platform: ruby
 authors:
 - Epsagon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-29 00:00:00.000000000 Z
+date: 2021-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opentelemetry-api
@@ -92,6 +92,7 @@ extra_rdoc_files: []
 files:
 - lib/epsagon.rb
 - lib/epsagon_constants.rb
+- lib/exporter_extension.rb
 - lib/instrumentation/aws_sdk.rb
 - lib/instrumentation/aws_sdk_plugin.rb
 - lib/instrumentation/epsagon_faraday_middleware.rb