ddtrace 0.40.0 → 0.50.0

data/benchmarks/profiler_sample_loop.rb

@@ -0,0 +1,69 @@
+require 'benchmark/ips'
+require 'ddtrace'
+require 'pry'
+
+# This benchmark measures the performance of the main stack sampling loop of the profiler
+
+class ProfilerSampleLoopBenchmark
+  def create_profiler
+    Datadog.configure do |c|
+      # c.diagnostics.debug = true
+      c.profiling.enabled = true
+      c.tracer.transport_options = proc { |t| t.adapter :test }
+    end
+
+    # Stop background threads
+    Datadog.profiler.shutdown!
+
+    # Call collection directly
+    @stack_collector = Datadog.profiler.collectors.first
+    @recorder = @stack_collector.recorder
+  end
+
+  def thread_with_very_deep_stack(depth: 500)
+    deep_stack = proc do |n|
+      if n > 0
+        deep_stack.call(n - 1)
+      else
+        sleep
+      end
+    end
+
+    Thread.new { deep_stack.call(depth) }.tap { |t| t.name = "Deep stack #{depth}" }
+  end
+
+  def run_benchmark
+    Benchmark.ips do |x|
+      x.config(time: 10, warmup: 2)
+
+      x.report("stack collector #{ENV['CONFIG']}") do
+        @stack_collector.collect_and_wait
+      end
+
+      x.save! 'profiler-sample-loop-results'
+      x.compare!
+    end
+
+    @recorder.flush
+  end
+
+  def run_forever
+    while true
+      1000.times { @stack_collector.collect_and_wait }
+      @recorder.flush
+      print '.'
+    end
+  end
+end
+
+puts "Current pid is #{Process.pid}"
+
+ProfilerSampleLoopBenchmark.new.instance_exec do
+  create_profiler
+  4.times { thread_with_very_deep_stack }
+  if ARGV.include?('--forever')
+    run_forever
+  else
+    run_benchmark
+  end
+end

data/benchmarks/profiler_submission.rb

@@ -0,0 +1,93 @@
+require 'benchmark/ips'
+require 'ddtrace'
+require 'pry'
+
+# This benchmark measures the performance of encoding pprofs and trying to submit them
+#
+# The FLUSH_DUMP_FILE (by default benchmarks/data/profiler-submission-marshal.gz, gathered from benchmarking using
+# the discourse forum rails app) can be generated by changing the scheduler.rb#flush_events to dump the contents of
+# "flush" to a file during a benchmark execution:
+#
+# dump_file = "marshal-#{Time.now.utc.to_i}.dump"
+# File.open(dump_file, "w") { |f| Marshal.dump(flush, f) }
+# Datadog.logger.info("Dumped to #{dump_file}")
+#
+# And then gzipping the result. (This can probably be automated a bit by adding an extra exporter, but the above worked
+# for me).
+
+class ProfilerSubmission
+  def create_profiler
+    @adapter_buffer = []
+
+    Datadog.configure do |c|
+      # c.diagnostics.debug = true
+      c.profiling.enabled = true
+      c.tracer.transport_options = proc { |t| t.adapter :test, @adapter_buffer }
+    end
+
+    # Stop background threads
+    Datadog.profiler.shutdown!
+
+    # Call exporter directly
+    @exporter = Datadog.profiler.scheduler.exporters.first
+    @flush = Marshal.load(
+      Zlib::GzipReader.new(File.open(ENV['FLUSH_DUMP_FILE'] || 'benchmarks/data/profiler-submission-marshal.gz'))
+    )
+  end
+
+  def check_valid_pprof
+    output_pprof = @adapter_buffer.last[:form]["data[0]"].io
+
+    expected_hashes = [
+      "75c65dec2d5d750faca3e905486c39ba44219636ac1bea15d5d624b741d4b62a",
+      "6780c47b3e271f2abe0346fd40ac358f6a34270b3a8f5743aacd1970bbbbc6f5"
+    ]
+    current_hash = Digest::SHA256.hexdigest(Zlib::GzipReader.new(output_pprof).read)
+
+    if expected_hashes.include?(current_hash)
+      puts "Output hash #{current_hash} matches known signature"
+    else
+      puts "WARNING: Unexpected pprof output -- unknown hash. Hashes seem to differ due to some of our dependencies changing, " \
+        "but it can also indicate that encoding output has become corrupted."
+    end
+  end
+
+  def run_benchmark
+    Benchmark.ips do |x|
+      x.config(time: 10, warmup: 2)
+
+      x.report("exporter #{ENV['CONFIG']}") do
+        run_once
+      end
+
+      x.save! 'profiler-submission-results.json'
+      x.compare!
+    end
+  end
+
+  def run_forever
+    while true
+      run_once
+      print '.'
+    end
+  end
+
+  def run_once
+    @adapter_buffer.clear
+    @exporter.export(@flush)
+  end
+end
+
+puts "Current pid is #{Process.pid}"
+
+ProfilerSubmission.new.instance_exec do
+  create_profiler
+  run_once
+  check_valid_pprof
+
+  if ARGV.include?('--forever')
+    run_forever
+  else
+    run_benchmark
+  end
+end

data/bin/ddtracerb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+require 'ddtrace/tasks/exec'
+require 'ddtrace/tasks/help'
+
+command = ARGV.shift
+
+case command
+when 'exec'
+  Datadog::Tasks::Exec.new(ARGV).run
+when 'help', '--help'
+  Datadog::Tasks::Help.new.run
+else
+  puts "Command '#{command}' is not valid for ddtrace."
+  Datadog::Tasks::Help.new.run
+end

data/ddtrace.gemspec

@@ -7,7 +7,7 @@ require 'ddtrace/version'
 Gem::Specification.new do |spec|
   spec.name                  = 'ddtrace'
   spec.version               = Datadog::VERSION::STRING
-  spec.required_ruby_version = ">= #{Datadog::VERSION::MINIMUM_RUBY_VERSION}"
+  spec.required_ruby_version = [">= #{Datadog::VERSION::MINIMUM_RUBY_VERSION}", "< #{Datadog::VERSION::MAXIMUM_RUBY_VERSION}"]
   spec.required_rubygems_version = '>= 2.0.0'
   spec.authors               = ['Datadog, Inc.']
   spec.email                 = ['dev@datadoghq.com']
@@ -29,44 +29,16 @@ Gem::Specification.new do |spec|
   end
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables   = ['ddtracerb']
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'msgpack'
-
-  # Optional extensions
-  # TODO: Move this to Appraisals?
-  spec.add_development_dependency 'dogstatsd-ruby', '>= 3.3.0'
-  spec.add_development_dependency 'opentracing', '>= 0.4.1'
-
-  # Development dependencies
-  spec.add_development_dependency 'rake', '>= 10.5'
-  spec.add_development_dependency 'rubocop', '= 0.49.1' if RUBY_VERSION >= '2.1.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rspec-collection_matchers', '~> 1.1'
-  spec.add_development_dependency 'minitest', '= 5.10.1'
-  spec.add_development_dependency 'minitest-around', '0.5.0'
-  spec.add_development_dependency 'minitest-stub_any_instance', '1.0.2'
-  spec.add_development_dependency 'appraisal', '~> 2.2'
-  spec.add_development_dependency 'yard', '~> 0.9'
-  spec.add_development_dependency 'webmock', '~> 2.0'
-  spec.add_development_dependency 'builder'
-  if RUBY_PLATFORM != 'java'
-    spec.add_development_dependency 'sqlite3', '~> 1.3.6'
+  if RUBY_VERSION >= '2.2.0'
+    spec.add_dependency 'msgpack'
   else
-    spec.add_development_dependency 'jdbc-sqlite3', '~> 3'
+    # msgpack 1.4 fails for Ruby 2.1: https://github.com/msgpack/msgpack-ruby/issues/205
+    spec.add_dependency 'msgpack', '< 1.4'
   end
-  spec.add_development_dependency 'climate_control', '~> 0.2.0'
 
-  # locking transitive dependency of webmock
-  spec.add_development_dependency 'addressable', '~> 2.4.0'
-  spec.add_development_dependency 'benchmark-ips', '~> 2.8'
-  spec.add_development_dependency 'benchmark-memory', '~> 0.1'
-  spec.add_development_dependency 'memory_profiler', '~> 0.9'
-  spec.add_development_dependency 'redcarpet', '~> 3.4' if RUBY_PLATFORM != 'java'
-  spec.add_development_dependency 'pry', '~> 0.10.4'
-  spec.add_development_dependency 'pry-stack_explorer', '~> 0.4.9.2'
-  spec.add_development_dependency 'simplecov', '~> 0.17'
-  spec.add_development_dependency 'warning', '~> 1' if RUBY_VERSION >= '2.5.0'
+  # Used by the profiler
+  spec.add_dependency 'ffi', '~> 1.0'
 end

data/docker-compose.yml

@@ -1,8 +1,8 @@
 version: '3.2'
 services:
   # MRI
-  tracer-2.0:
-    image: palazzem/docker-library:ddtrace_rb_2_0_0
+  tracer-2.1:
+    image: palazzem/docker-library:ddtrace_rb_2_1_10
     command: /bin/bash
     depends_on:
       - ddagent
@@ -24,15 +24,15 @@ services:
       - TEST_MYSQL_HOST=mysql
       - TEST_POSTGRES_HOST=postgres
       - TEST_PRESTO_HOST=presto
+      - TEST_PRESTO_PORT=8080
       - TEST_REDIS_HOST=redis
     stdin_open: true
     tty: true
     volumes:
       - .:/app
-      - bundle-2.0:/usr/local/bundle
-      - gemfiles-2.0:/app/gemfiles
-  tracer-2.1:
-    image: palazzem/docker-library:ddtrace_rb_2_1_10
+      - bundle-2.1:/usr/local/bundle
+  tracer-2.2:
+    image: palazzem/docker-library:ddtrace_rb_2_2_10
     command: /bin/bash
     depends_on:
       - ddagent
@@ -60,10 +60,9 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.1:/usr/local/bundle
-      - gemfiles-2.1:/app/gemfiles
-  tracer-2.2:
-    image: palazzem/docker-library:ddtrace_rb_2_2_10
+      - bundle-2.2:/usr/local/bundle
+  tracer-2.3:
+    image: palazzem/docker-library:ddtrace_rb_2_3_8
     command: /bin/bash
     depends_on:
       - ddagent
@@ -91,10 +90,9 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.2:/usr/local/bundle
-      - gemfiles-2.2:/app/gemfiles
-  tracer-2.3:
-    image: palazzem/docker-library:ddtrace_rb_2_3_8
+      - bundle-2.3:/usr/local/bundle
+  tracer-2.4:
+    image: palazzem/docker-library:ddtrace_rb_2_4_6
     command: /bin/bash
     depends_on:
       - ddagent
@@ -122,10 +120,9 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.3:/usr/local/bundle
-      - gemfiles-2.3:/app/gemfiles
-  tracer-2.4:
-    image: palazzem/docker-library:ddtrace_rb_2_4_6
+      - bundle-2.4:/usr/local/bundle
+  tracer-2.5:
+    image: marcotc/docker-library:ddtrace_rb_2_5_6
     command: /bin/bash
     depends_on:
       - ddagent
@@ -153,10 +150,9 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.4:/usr/local/bundle
-      - gemfiles-2.4:/app/gemfiles
-  tracer-2.5:
-    image: marcotc/docker-library:ddtrace_rb_2_5_6
+      - bundle-2.5:/usr/local/bundle
+  tracer-2.6:
+    image: marcotc/docker-library:ddtrace_rb_2_6_4
     command: /bin/bash
     depends_on:
       - ddagent
@@ -184,10 +180,9 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.5:/usr/local/bundle
-      - gemfiles-2.5:/app/gemfiles
-  tracer-2.6:
-    image: marcotc/docker-library:ddtrace_rb_2_6_4
+      - bundle-2.6:/usr/local/bundle
+  tracer-2.7:
+    image: marcotc/docker-library:ddtrace_rb_2_7_0
     command: /bin/bash
     depends_on:
       - ddagent
@@ -215,10 +210,37 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.6:/usr/local/bundle
-      - gemfiles-2.6:/app/gemfiles
-  tracer-2.7:
-    image: marcotc/docker-library:ddtrace_rb_2_7_0
+      - bundle-2.7:/usr/local/bundle
+  tracer-3.0:
+    image: marcotc/docker-library:ddtrace_rb_3.0.0
+    command: /bin/bash
+    depends_on:
+      - ddagent
+      - elasticsearch
+      - memcached
+      - mongodb
+      - mysql
+      - postgres
+      - redis
+    env_file: ./.env
+    environment:
+      - BUNDLE_GEMFILE=/app/Gemfile
+      - DD_AGENT_HOST=ddagent
+      - TEST_DATADOG_INTEGRATION=1
+      - TEST_ELASTICSEARCH_HOST=elasticsearch
+      - TEST_MEMCACHED_HOST=memcached
+      - TEST_MONGODB_HOST=mongodb
+      - TEST_MYSQL_HOST=mysql
+      - TEST_POSTGRES_HOST=postgres
+      - TEST_REDIS_HOST=redis
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+      - bundle-3.0:/usr/local/bundle
+  # JRuby
+  tracer-jruby-9.2.0.0:
+    image: marcotc/docker-library:ddtrace_rb_jruby_9_2_0_0
     command: /bin/bash
     depends_on:
       - ddagent
@@ -246,11 +268,40 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-2.7:/usr/local/bundle
-      - gemfiles-2.7:/app/gemfiles
-  # JRuby
-  tracer-jruby-9.2:
-    image: marcotc/docker-library:ddtrace_rb_jruby_9_2
+      - bundle-jruby-9.2.0.0:/usr/local/bundle
+  tracer-jruby-9.2-latest:
+    image: marcotc/docker-library:ddtrace_rb_jruby_9_2_11_1
+    command: /bin/bash
+    depends_on:
+      - ddagent
+      - elasticsearch
+      - memcached
+      - mongodb
+      - mysql
+      - postgres
+      - presto
+      - redis
+    env_file: ./.env
+    environment:
+      - BUNDLE_GEMFILE=/app/Gemfile
+      - DD_AGENT_HOST=ddagent
+      - TEST_DATADOG_INTEGRATION=1
+      - TEST_ELASTICSEARCH_HOST=elasticsearch
+      - TEST_MEMCACHED_HOST=memcached
+      - TEST_MONGODB_HOST=mongodb
+      - TEST_MYSQL_HOST=mysql
+      - TEST_POSTGRES_HOST=postgres
+      - TEST_PRESTO_HOST=presto
+      - TEST_PRESTO_PORT=8080
+      - TEST_REDIS_HOST=redis
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+      - bundle-jruby-9.2-latest:/usr/local/bundle
+  # TruffleRuby
+  tracer-truffleruby-21.1.0:
+    image: ivoanjo/docker-library:ddtrace_rb_truffleruby_21_1_0
     command: /bin/bash
     depends_on:
       - ddagent
@@ -278,8 +329,7 @@ services:
     tty: true
     volumes:
       - .:/app
-      - bundle-jruby-9.2:/usr/local/bundle
-      - gemfiles-jruby-9.2:/app/gemfiles
+      - bundle-truffleruby-21.1.0:/usr/local/bundle
   ddagent:
     image: datadog/agent
     environment:
@@ -338,7 +388,8 @@ services:
     ports:
       - "${TEST_POSTGRES_PORT}:5432"
   presto:
-    image:  prestosql/presto
+    # Move to trinodb/trino after https://github.com/treasure-data/presto-client-ruby/issues/64 is resolved.
+    image: starburstdata/presto:332-e.9
     expose:
       - "8080"
     ports:
@@ -350,7 +401,6 @@ services:
     ports:
       - "${TEST_REDIS_PORT}:6379"
 volumes:
-  bundle-2.0:
   bundle-2.1:
   bundle-2.2:
   bundle-2.3:
@@ -358,13 +408,7 @@ volumes:
   bundle-2.5:
   bundle-2.6:
   bundle-2.7:
-  bundle-jruby-9.2:
-  gemfiles-2.0:
-  gemfiles-2.1:
-  gemfiles-2.2:
-  gemfiles-2.3:
-  gemfiles-2.4:
-  gemfiles-2.5:
-  gemfiles-2.6:
-  gemfiles-2.7:
-  gemfiles-jruby-9.2:
+  bundle-3.0:
+  bundle-jruby-9.2.0.0:
+  bundle-jruby-9.2-latest:
+  bundle-truffleruby-21.1.0: