ddtrace 0.46.0 → 0.47.0

data/Gemfile

@@ -16,9 +16,15 @@ gem 'minitest', '= 5.10.1'
 gem 'minitest-around', '0.5.0'
 gem 'minitest-stub_any_instance', '1.0.2'
 gem 'pimpmychangelog', '>= 0.1.2'
-gem 'pry', '~> 0.12.2'
-gem 'pry-nav', '~> 0.3.0'
-gem 'pry-stack_explorer', '~> 0.4.9' if RUBY_PLATFORM != 'java'
+gem 'pry'
+if RUBY_PLATFORM != 'java'
+  # There's a few incompatibilities between pry/pry-byebug on older Rubies
+  gem 'pry-byebug' if RUBY_VERSION >= '2.6.0' && RUBY_ENGINE != 'truffleruby'
+  gem 'pry-nav' if RUBY_VERSION < '2.6.0'
+  gem 'pry-stack_explorer'
+else
+  gem 'pry-debugger-jruby'
+end
 gem 'rake', '>= 10.5'
 gem 'redcarpet', '~> 3.4' if RUBY_PLATFORM != 'java'
 gem 'rspec', '~> 3.10'
@@ -26,7 +32,16 @@ gem 'rspec-collection_matchers', '~> 1.1'
 gem 'rspec_junit_formatter', '>= 0.4.1'
 gem 'rspec_n', '~> 1.3' if RUBY_VERSION >= '2.3.0'
 gem 'ruby-prof', '~> 1.4' if RUBY_PLATFORM != 'java' && RUBY_VERSION >= '2.4.0'
-gem 'simplecov', '~> 0.17'
+if RUBY_VERSION >= '2.5.0'
+  # Merging branch coverage results does not work for old, unsupported rubies.
+  # We have a fix up for review, https://github.com/simplecov-ruby/simplecov/pull/972,
+  # but given it only affects unsupported version of Ruby, it might not get merged.
+  gem 'simplecov', git: 'https://github.com/DataDog/simplecov', ref: '3bb6b7ee58bf4b1954ca205f50dd44d6f41c57db'
+else
+  # Compatible with older rubies. This version still produces compatible output
+  # with a newer version when the reports are merged.
+  gem 'simplecov', '~> 0.17'
+end
 gem 'warning', '~> 1' if RUBY_VERSION >= '2.5.0'
 gem 'webmock', '>= 3.10.0'
 gem 'webrick', '>= 1.7.0' if RUBY_VERSION >= '3.0.0' # No longer bundled by default since Ruby 3.0

data/LICENSE-3rdparty.csv

@@ -0,0 +1,2 @@
+Component,Origin,License,Copyright
+ddtrace/vendor/multipart-post,https://github.com/socketry/multipart-post,MIT,"Copyright (c) 2007-2013 Nick Sieger."

data/docker-compose.yml

@@ -277,8 +277,8 @@ services:
       - bundle-3.0:/usr/local/bundle
       - gemfiles-3.0:/app/gemfiles
   # JRuby
-  tracer-jruby-9.2:
-    image: marcotc/docker-library:ddtrace_rb_jruby_9_2
+  tracer-jruby-9.2.0.0:
+    image: marcotc/docker-library:ddtrace_rb_jruby_9_2_0_0
     command: /bin/bash
     depends_on:
       - ddagent
@@ -306,8 +306,71 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-jruby-9.2:/usr/local/bundle
-      - gemfiles-jruby-9.2:/app/gemfiles
+      - bundle-jruby-9.2.0.0:/usr/local/bundle
+      - gemfiles-jruby-9.2.0.0:/app/gemfiles
+  tracer-jruby-9.2-latest:
+    image: marcotc/docker-library:ddtrace_rb_jruby_9_2_11_1
+    command: /bin/bash
+    depends_on:
+      - ddagent
+      - elasticsearch
+      - memcached
+      - mongodb
+      - mysql
+      - postgres
+      - presto
+      - redis
+    env_file: ./.env
+    environment:
+      - BUNDLE_GEMFILE=/app/Gemfile
+      - DD_AGENT_HOST=ddagent
+      - TEST_DATADOG_INTEGRATION=1
+      - TEST_ELASTICSEARCH_HOST=elasticsearch
+      - TEST_MEMCACHED_HOST=memcached
+      - TEST_MONGODB_HOST=mongodb
+      - TEST_MYSQL_HOST=mysql
+      - TEST_POSTGRES_HOST=postgres
+      - TEST_PRESTO_HOST=presto
+      - TEST_PRESTO_PORT=8080
+      - TEST_REDIS_HOST=redis
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+      - bundle-jruby-9.2-latest:/usr/local/bundle
+      - gemfiles-jruby-9.2-latest:/app/gemfiles
+  # TruffleRuby
+  tracer-truffleruby-21.0.0:
+    image: ivoanjo/docker-library:ddtrace_rb_truffleruby_21_0_0
+    command: /bin/bash
+    depends_on:
+      - ddagent
+      - elasticsearch
+      - memcached
+      - mongodb
+      - mysql
+      - postgres
+      - presto
+      - redis
+    env_file: ./.env
+    environment:
+      - BUNDLE_GEMFILE=/app/Gemfile
+      - DD_AGENT_HOST=ddagent
+      - TEST_DATADOG_INTEGRATION=1
+      - TEST_ELASTICSEARCH_HOST=elasticsearch
+      - TEST_MEMCACHED_HOST=memcached
+      - TEST_MONGODB_HOST=mongodb
+      - TEST_MYSQL_HOST=mysql
+      - TEST_POSTGRES_HOST=postgres
+      - TEST_PRESTO_HOST=presto
+      - TEST_PRESTO_PORT=8080
+      - TEST_REDIS_HOST=redis
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+      - bundle-truffleruby-21.0.0:/usr/local/bundle
+      - gemfiles-truffleruby-21.0.0:/app/gemfiles
   ddagent:
     image: datadog/agent
     environment:
@@ -366,7 +429,8 @@ services:
     ports:
       - "${TEST_POSTGRES_PORT}:5432"
   presto:
-    image:  prestosql/presto
+    # Move to trinodb/trino after https://github.com/treasure-data/presto-client-ruby/issues/64 is resolved.
+    image: starburstdata/presto:332-e.9
     expose:
       - "8080"
     ports:
@@ -387,7 +451,9 @@ volumes:
   bundle-2.6:
   bundle-2.7:
   bundle-3.0:
-  bundle-jruby-9.2:
+  bundle-jruby-9.2.0.0:
+  bundle-jruby-9.2-latest:
+  bundle-truffleruby-21.0.0:
   gemfiles-2.0:
   gemfiles-2.1:
   gemfiles-2.2:
@@ -397,4 +463,6 @@ volumes:
   gemfiles-2.6:
   gemfiles-2.7:
   gemfiles-3.0:
-  gemfiles-jruby-9.2:
+  gemfiles-jruby-9.2.0.0:
+  gemfiles-jruby-9.2-latest:
+  gemfiles-truffleruby-21.0.0:

data/docs/GettingStarted.md

@@ -46,6 +46,7 @@ To contribute, check out the [contribution guidelines][contribution docs] and [d
      - [gRPC](#grpc)
      - [http.rb](#http-rb)
      - [httpclient](#httpclient)
+     - [httpx](#httpx)
      - [MongoDB](#mongodb)
      - [MySQL2](#mysql2)
      - [Net/HTTP](#net-http)
@@ -398,6 +399,7 @@ For a list of available integrations, and their configuration options, please re
 | gRPC                     | `grpc`                     | `>= 1.7`                 | *gem not available*       | *[Link](#grpc)*                     | *[Link](https://github.com/grpc/grpc/tree/master/src/rubyc)*                   |
 | http.rb                  | `httprb`                   | `>= 2.0`                 | `>= 2.0`                  | *[Link](#http-rb)*                  | *[Link](https://github.com/httprb/http)*                                       |
 | httpclient                | `httpclient`              | `>= 2.2`                 | `>= 2.2`                  | *[Link](#httpclient)*               | *[Link](https://github.com/nahi/httpclient)*                                     |
+| httpx                     | `httpx`                   | `>= 0.11`                | `>= 0.11`                 | *[Link](#httpx)*                    | *[Link](https://gitlab.com/honeyryderchuck/httpx)*                             |
 | Kafka                    | `ruby-kafka`               | `>= 0.7.10`              | `>= 0.7.10`               | *[Link](#kafka)*                    | *[Link](https://github.com/zendesk/ruby-kafka)*                                |
 | MongoDB                  | `mongo`                    | `>= 2.1`                 | `>= 2.1`                  | *[Link](#mongodb)*                  | *[Link](https://github.com/mongodb/mongo-ruby-driver)*                         |
 | MySQL2                   | `mysql2`                   | `>= 0.3.21`              | *gem not available*       | *[Link](#mysql2)*                   | *[Link](https://github.com/brianmario/mysql2)*                                 |
@@ -546,17 +548,20 @@ Datadog.configure do |c|
   # Symbol matching your database connection in config/database.yml
   # Only available if you are using Rails with ActiveRecord.
   c.use :active_record, describes: :secondary_database, service_name: 'secondary-db'
-
+  
+  # Block configuration pattern.
   c.use :active_record, describes: :secondary_database do |second_db|
     second_db.service_name = 'secondary-db'
   end
 
   # Connection string with the following connection settings:
-  # Adapter, user, host, port, database
+  # adapter, username, host, port, database
+  # Other fields are ignored.
   c.use :active_record, describes: 'mysql2://root@127.0.0.1:3306/mysql', service_name: 'secondary-db'
 
-  # Hash with following connection settings
-  # Adapter, user, host, port, database
+  # Hash with following connection settings:
+  # adapter, username, host, port, database
+  # Other fields are ignored.
   c.use :active_record, describes: {
       adapter:  'mysql2',
       host:     '127.0.0.1',
@@ -568,6 +573,27 @@ Datadog.configure do |c|
 end
 ```
 
+You can also create configurations based on partial matching of database connection fields:
+
+```ruby
+Datadog.configure do |c|
+  # Matches any connection on host `127.0.0.1`.
+  c.use :active_record, describes: { host:  '127.0.0.1' }, service_name: 'local-db'
+
+  # Matches any `mysql2` connection.
+  c.use :active_record, describes: { adapter: 'mysql2'}, service_name: 'mysql-db'
+  
+  # Matches any `mysql2` connection to the `reports` database.
+  #
+  # In case of multiple matching `describe` configurations, the latest one applies.
+  # In this case a connection with both adapter `mysql` and database `reports`
+  # will be configured `service_name: 'reports-db'`, not `service_name: 'mysql-db'`.
+  c.use :active_record, describes: { adapter: 'mysql2', database:  'reports'}, service_name: 'reports-db'
+end
+```
+
+When multiple `describes` configurations match a connection, the latest configured rule that matches will be applied.
+
 If ActiveRecord traces an event that uses a connection that matches a key defined by `describes`, it will use the trace settings assigned to that connection. If the connection does not match any of the described connections, it will use default settings defined by `c.use :active_record` instead.
 
 ### Active Support
@@ -1078,6 +1104,25 @@ Where `options` is an optional `Hash` that accepts the following parameters:
 | `service_name` | Service name for `httpclient` instrumentation. | `'httpclient'` |
 | `split_by_domain` | Uses the request domain as the service name when set to `true`. | `false` |
 
+### httpx
+
+`httpx` maintains its [own integration with `ddtrace`](https://honeyryderchuck.gitlab.io/httpx/wiki/Datadog-Adapter):
+
+```ruby
+require "ddtrace"
+require "httpx/adapters/datadog"
+
+Datadog.configure do |c|
+  c.use :httpx
+
+  # optionally, specify a different service name for hostnames matching a regex
+  c.use :httpx, describes: /user-[^.]+\.example\.com/ do |http|
+    http.service_name = 'user.example.com'
+    http.split_by_domain = false # Only necessary if split_by_domain is true by default
+  end
+end
+```
+
 ### Kafka
 
 The Kafka integration provides tracing of the `ruby-kafka` gem:
@@ -1537,12 +1582,17 @@ Datadog.configure do |c|
   # The default configuration for any redis client
   c.use :redis, service_name: 'redis-default'
 
-  # The configuration matching a given unix socket
+  # The configuration matching a given unix socket.
   c.use :redis, describes: { url: 'unix://path/to/file' }, service_name: 'redis-unix'
 
-  # Connection string
-  c.use :redis, describes: { url: 'redis://127.0.0.1:6379/0' }, service_name: 'redis-connection-string'
-  # Client host, port, db, scheme
+  # For network connections, only these fields are considered during matching:
+  # scheme, host, port, db
+  # Other fields are ignored.
+  
+  # Network connection string
+  c.use :redis, describes: 'redis://127.0.0.1:6379/0', service_name: 'redis-connection-string'
+  c.use :redis, describes: { url: 'redis://127.0.0.1:6379/1' }, service_name: 'redis-connection-url'
+  # Network client hash
   c.use :redis, describes: { host: 'my-host.com', port: 6379, db: 1, scheme: 'redis' }, service_name: 'redis-connection-hash'
   # Only a subset of the connection hash
   c.use :redis, describes: { host: ENV['APP_CACHE_HOST'], port: ENV['APP_CACHE_PORT'] }, service_name: 'redis-cache'
@@ -1550,6 +1600,8 @@ Datadog.configure do |c|
 end
 ```
 
+When multiple `describes` configurations match a connection, the latest configured rule that matches will be applied.
+
 ### Resque
 
 The Resque integration uses Resque hooks that wraps the `perform` method.
@@ -2087,6 +2139,7 @@ For more details on how to activate distributed tracing for integrations, see th
 - [Sinatra](#sinatra)
 - [http.rb](#http-rb)
 - [httpclient](#httpclient)
+- [httpx](#httpx)
 
 **Using the HTTP propagator**
 

data/lib/ddtrace/configuration.rb

@@ -1,5 +1,4 @@
 require 'forwardable'
-
 require 'ddtrace/configuration/pin_setup'
 require 'ddtrace/configuration/settings'
 require 'ddtrace/configuration/components'
@@ -9,6 +8,33 @@ module Datadog
   module Configuration
     extend Forwardable
 
+    # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+    #
+    # This is important because components can end up being accessed from multiple application threads (for instance on
+    # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+    #
+    # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+    # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+    #
+    # Every method that directly or indirectly mutates @components should be holding the lock (through
+    # #safely_synchronize) while doing so.
+    COMPONENTS_WRITE_LOCK = Mutex.new
+    private_constant :COMPONENTS_WRITE_LOCK
+
+    # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+    # This was needed because we ran into several issues where we still needed to read the old
+    # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+    # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+    #
+    # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+    # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+    # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+    # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+    # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+    # Concurrency is hard.
+    COMPONENTS_READ_LOCK = Mutex.new
+    private_constant :COMPONENTS_READ_LOCK
+
     attr_writer :configuration
 
     def configuration
@@ -19,13 +45,15 @@ module Datadog
       if target.is_a?(Settings)
         yield(target) if block_given?
 
-        # Build immutable components from settings
-        @components ||= nil
-        @components = if @components
-                        replace_components!(target, @components)
-                      else
-                        build_components(target)
-                      end
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(target, @components)
+            else
+              build_components(target)
+            end
+          )
+        end
 
         target
       else
@@ -40,18 +68,14 @@ module Datadog
       :tracer
 
     def logger
-      if instance_variable_defined?(:@components) && @components
+      # avoid initializing components if they didn't already exist
+      current_components = components(allow_initialization: false)
+
+      if current_components
         @temp_logger = nil
-        components.logger
+        current_components.logger
       else
-        # Use default logger without initializing components.
-        # This prevents recursive loops while initializing.
-        # e.g. Get logger --> Build components --> Log message --> Repeat...
-        @temp_logger ||= begin
-          logger = configuration.logger.instance || Datadog::Logger.new($stdout)
-          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
-          logger
-        end
+        logger_without_components
       end
     end
 
@@ -65,7 +89,9 @@ module Datadog
     #
     # Components won't be automatically reinitialized after a shutdown.
     def shutdown!
-      components.shutdown! if instance_variable_defined?(:@components) && @components
+      safely_synchronize do
+        @components.shutdown! if components?
+      end
     end
 
     # Gracefully shuts down the tracer and disposes of component references,
@@ -74,18 +100,51 @@ module Datadog
     # In contrast with +#shutdown!+, components will be automatically
     # reinitialized after a reset.
     def reset!
-      shutdown!
-      @components = nil
+      safely_synchronize do |write_components|
+        @components.shutdown! if components?
+        write_components.call(nil)
+      end
     end
 
     protected
 
-    def components
-      @components ||= build_components(configuration)
+    def components(allow_initialization: true)
+      current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+      return current_components if current_components || !allow_initialization
+
+      safely_synchronize do |write_components|
+        (defined?(@components) && @components) || write_components.call(build_components(configuration))
+      end
     end
 
     private
 
+    def safely_synchronize
+      # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+      # safely_synchronize, this forces all writers to go through this method.
+      write_components = proc do |new_value|
+        COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+      end
+
+      COMPONENTS_WRITE_LOCK.synchronize do
+        begin
+          yield write_components
+        rescue ThreadError => e
+          logger_without_components.error(
+            'Detected deadlock during ddtrace initialization. ' \
+            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+            "\n\tSource:\n\t#{e.backtrace.join("\n\t")}"
+          )
+          nil
+        end
+      end
+    end
+
+    def components?
+      # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+      (defined?(@components) && @components) != nil
+    end
+
     def build_components(settings)
       components = Components.new(settings)
       components.startup!(settings)
@@ -99,5 +158,15 @@ module Datadog
       components.startup!(settings)
       components
     end
+
+    def logger_without_components
+      # Use default logger without initializing components.
+      # This enables logging during initialization, otherwise we'd run into deadlocks.
+      @temp_logger ||= begin
+        logger = configuration.logger.instance || Datadog::Logger.new($stdout)
+        logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+        logger
+      end
+    end
   end
 end