ddtrace 0.13.0 → 0.47.0

data/integration/README.md

@@ -0,0 +1,68 @@
+# Datadog APM Ruby integration test suite
+
+Integration tests for `ddtrace` that use a variety of real applications.
+
+## Quickstart
+
+1. Build Docker base images:
+
+    ```sh
+    #!/bin/bash
+    ./script/build-images
+    ```
+
+2. Choose an application and follow instructions (in corresponding `README.md`.)
+
+## Demo applications
+
+Ruby demo applications are configured with Datadog APM, which can be used to generate sample traces/profiles. These are used to drive tests in the integration suite.
+
+### Applications
+
+See `README.md` in each directory for more information:
+
+- `apps/rack`: Rack application
+- `apps/rails-five`: Rails 5 application
+
+### Base images
+
+The `images/` folders hosts some images for Ruby applications.
+
+- `datadog/dd-apm-demo:wrk` / `images/wrk/Dockerfile`: `wrk` load testing application (for generating load)
+- `datadog/dd-apm-demo:agent` / `images/agent/Dockerfile`: Datadog agent (with default configuration)
+- `datadog/dd-apm-demo:rb-<RUBY_VERSION>` / `images/<RUBY_VERSION>/Dockerfile`: MRI Ruby & `Datadog::DemoEnv` (where `<RUBY_VERSION>` is minor version e.g. `2.7`)
+
+Ruby base images include `Datadog::DemoEnv` and other helpers.
+
+### Debugging
+
+#### Profiling memory
+
+Create a memory heap dump via:
+
+```sh
+# Profile for 5 minutes, dump heap to /data/app/ruby-heap.dump
+# Where PID = process ID
+bundle exec rbtrace -p PID -e 'Thread.new{GC.start; require "objspace"; ObjectSpace.trace_object_allocations_start; sleep(300); io=File.open("/data/app/ruby-heap.dump", "w"); ObjectSpace.dump_all(output: io); io.close}'
+```
+
+Then analyze it using `analyzer.rb` (built into the Ruby base images) with:
+
+```sh
+# Group objects by generation
+ruby /vendor/dd-demo/datadog/analyzer.rb /data/app/ruby-heap.dump
+
+# List objects in GEN_NUM, group by source location
+ruby /vendor/dd-demo/datadog/analyzer.rb /data/app/ruby-heap.dump GEN_NUM
+
+# List objects in all generations, group by source location, descending.
+ruby /vendor/dd-demo/datadog/analyzer.rb /data/app/ruby-heap.dump objects
+
+# List objects in all generations, group by source location, descending.
+# Up to generation END_GEN.
+ruby /vendor/dd-demo/datadog/analyzer.rb /data/app/ruby-heap.dump objects END_GEN
+
+# List objects in all generations, group by source location, descending.
+# Between generations START_GEN to END_GEN inclusive.
+ruby /vendor/dd-demo/datadog/analyzer.rb /data/app/ruby-heap.dump objects END_GEN START_GEN
+```

data/integration/apps/rack/.dockerignore

@@ -0,0 +1 @@
+Gemfile.lock

data/integration/apps/rack/.envrc.sample

@@ -0,0 +1 @@
+export DD_API_KEY=<Your Datadog API key here>

data/integration/apps/rack/.gitignore

@@ -0,0 +1,4 @@
+# Ignore local variables
+.envrc
+.byebug_history
+data

data/integration/apps/rack/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/integration/apps/rack/Dockerfile

@@ -0,0 +1,25 @@
+# Select base image
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+# Setup directory
+RUN mkdir /app
+WORKDIR /app
+
+# Setup specific version of ddtrace, if specified.
+ARG ddtrace_git
+ENV DD_DEMO_ENV_GEM_GIT_DDTRACE ${ddtrace_git}
+
+ARG ddtrace_ref
+ENV DD_DEMO_ENV_GEM_REF_DDTRACE ${ddtrace_ref}
+
+# Install dependencies
+COPY Gemfile /app/Gemfile
+RUN bundle install
+
+# Add files
+COPY . /app
+
+# Set entrypoint
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD ["bin/setup && bin/run"]

data/integration/apps/rack/Dockerfile-ci

@@ -0,0 +1,11 @@
+# Select base image
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+# Add gem
+COPY . /vendor/dd-trace-rb
+
+# Install dependencies
+# Setup specific version of ddtrace, if specified.
+ENV DD_DEMO_ENV_GEM_LOCAL_DDTRACE /vendor/dd-trace-rb
+RUN bundle install

data/integration/apps/rack/Gemfile

@@ -0,0 +1,24 @@
+require 'datadog/demo_env'
+
+source 'https://rubygems.org' do
+  gem 'puma'
+  gem 'unicorn'
+
+  gem 'dogstatsd-ruby'
+  gem 'rack'
+
+  gem 'ffi'
+  gem 'google-protobuf'
+
+  # Choose correct specs for 'ddtrace' demo environment
+  gem 'ddtrace', *Datadog::DemoEnv.gem_spec('ddtrace')
+
+  # Development
+  gem 'byebug'
+  # gem 'pry-stack_explorer', platform: :ruby
+  # gem 'rbtrace'
+  # gem 'ruby-prof'
+
+  # Testing/CI
+  gem 'rspec'
+end

data/integration/apps/rack/README.md

@@ -0,0 +1,93 @@
+# Rack: Demo application for Datadog APM
+
+A generic Rack web application with some common use scenarios.
+
+For generating Datadog APM traces and profiles.
+
+## Installation
+
+Install [direnv](https://github.com/direnv/direnv) for applying local settings.
+
+1. `cp .envrc.sample .envrc` and add your Datadog API key.
+2. `direnv allow` to load the env var.
+4. `docker-compose run --rm app bin/setup`
+
+## Running the application
+
+### To monitor performance of Docker containers with Datadog
+
+```sh
+docker run --rm --name dd-agent  -v /var/run/docker.sock:/var/run/docker.sock:ro -v /proc/:/host/proc/:ro -v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro -e API_KEY=$DD_API_KEY datadog/docker-dd-agent:latest
+```
+
+### Starting the web server
+
+Run `docker-compose up` to auto-start the webserver. It should bind to `localhost:80`.
+
+Alternatively, you can run it manually with:
+
+```sh
+docker-compose run --rm -p 80:80 app bin/run <process>
+```
+
+The `<process>` argument is optional, and will default to `DD_DEMO_ENV_PROCESS` if not provided. See [Processes](#processes) for more details.
+
+##### Processes
+
+Within the container, run `bin/run <process>` where `<process>` is one of the following values:
+
+ - `puma`: Puma web server
+ - `unicorn`: Unicorn web server
+ - `irb`: IRB session
+
+ Alternatively, set `DD_DEMO_ENV_PROCESS` to run a particular process by default when `bin/run` is run.
+
+##### Features
+
+Set `DD_DEMO_ENV_PROCESS` to a comma-delimited list of any of the following values to activate the feature:
+
+ - `tracing`: Tracing instrumentation
+ - `profiling`: Profiling (NOTE: Must also set `DD_PROFILING_ENABLED` to match.)
+ - `debug`: Enable diagnostic debug mode
+ - `analytics`: Enable trace analytics
+ - `runtime_metrics`: Enable runtime metrics
+ - `pprof_to_file`: Dump profiling pprof to file instead of agent.
+
+e.g. `DD_DEMO_ENV_PROCESS=tracing,profiling`
+
+##### Routes
+
+```sh
+# Health check
+curl -v localhost/health
+
+# Basic demo scenarios
+curl -v localhost/basic/fibonacci
+curl -v -XPOST localhost/basic/default
+
+# Job demo scenarios
+curl -v -XPOST localhost/jobs
+```
+
+### Load tester
+
+Docker configuration automatically creates and runs [Wrk](https://github.com/wg/wrk) load testing containers. By default it runs the `basic/default` scenario described in the `wrk` image to give a baseload.
+
+You can modify the `load-tester` container in `docker-compose.yml` to change the load type or scenario run. Set the container's `command` to any set of arguments `wrk` accepts.
+
+You can also define your own custom scenario by creating a LUA file, mounting it into the container, and passing it as an argument via `command`.
+
+### Running integration tests
+
+You can run integration tests using the following and substituting for the Ruby major and minor version (e.g. `2.7`)
+
+```sh
+./script/build-images -v <RUBY_VERSION>
+./script/ci -v <RUBY_VERSION>
+```
+
+Or inside a running container:
+
+```sh
+./bin/test
+```

data/integration/apps/rack/app/acme.rb

@@ -0,0 +1,80 @@
+require 'rack'
+
+module Acme
+  class Application
+    def call(env)
+      request = Rack::Request.new(env)
+      router.route!(request)
+    end
+
+    def router
+      Router.new(
+        '/' => { controller: controllers[:health], action: :check },
+        '/health' => { controller: controllers[:health], action: :check },
+        '/basic/fibonacci' => { controller: controllers[:basic], action: :fibonacci },
+        '/basic/default' => { controller: controllers[:basic], action: :default }
+      )
+    end
+
+    def controllers
+      {
+        basic: Controllers::Basic.new,
+        health: Controllers::Health.new
+      }
+    end
+  end
+
+  class Router
+    attr_reader :routes
+
+    def initialize(routes)
+      @routes = routes
+    end
+
+    def route!(request)
+      begin
+        if route = routes[request.path]
+          route[:controller].send(route[:action], request)
+        else
+          not_found(request)
+        end
+      rescue StandardError => e
+        application_error(request, e)
+      end
+    end
+
+    def not_found(request)
+      [404, { 'Content-Type' => 'text/plain' }, ["404 Not Found: #{request.path}"]]
+    end
+
+    def application_error(request, error)
+      [500, { 'Content-Type' => 'text/plain' }, ["500 Application Error: #{error.message} Location: #{error.backtrace.first(3)}"]]
+    end
+  end
+
+  module Controllers
+    class Basic
+      def fibonacci(request)
+        n = rand(25..35)
+        result = fib(n)
+        ['200', { 'Content-Type' => 'text/plain' }, ["Basic: Fibonacci(#{n}): #{result}"]]
+      end
+
+      def default(request)
+        ['200', { 'Content-Type' => 'text/plain' }, ['Basic: Default']]
+      end
+
+      private
+
+      def fib(n)
+        n <= 1 ? n : fib(n-1) + fib(n-2)
+      end
+    end
+
+    class Health
+      def check(request)
+        ['204', {}, []]
+      end
+    end
+  end
+end

data/integration/apps/rack/app/datadog.rb

@@ -0,0 +1,17 @@
+require 'datadog/demo_env'
+require 'ddtrace'
+
+Datadog.configure do |c|
+  c.diagnostics.debug = true if Datadog::DemoEnv.feature?('debug')
+  c.analytics_enabled = true if Datadog::DemoEnv.feature?('analytics')
+  c.runtime_metrics.enabled = true if Datadog::DemoEnv.feature?('runtime_metrics')
+
+  if Datadog::DemoEnv.feature?('tracing')
+    c.use :rack, service_name: 'acme-rack'
+  end
+
+  if Datadog::DemoEnv.feature?('pprof_to_file')
+    # Reconfigure transport to write pprof to file
+    c.profiling.exporter.transport = Datadog::DemoEnv.profiler_file_transport
+  end
+end

data/integration/apps/rack/bin/run

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+# Start application process
+puts "\n== Starting application process =="
+
+profiling = Datadog::DemoEnv.feature?('profiling') ? 'DD_PROFILING_ENABLED=true ddtracerb exec ' : ''
+process = (ARGV[0] || Datadog::DemoEnv.process)
+command = case process
+          when 'puma'
+            "bundle exec #{profiling}puma -C /app/config/puma.rb /app/config.ru"
+          when 'unicorn'
+            "bundle exec #{profiling}unicorn -c /app/config/unicorn.rb /app/config.ru"
+          when 'irb'
+            "bundle exec #{profiling}irb"
+          when nil, ''
+            abort("\n== ERROR: Must specify a application process! ==")
+          else
+            abort("\n== ERROR: Unknown application process '#{process}' ==")
+          end
+
+puts "Run: #{command}"
+Kernel.exec(command)

data/integration/apps/rack/bin/setup

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  puts "Run: #{args.join(' ')}"
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  puts "\n== Installing dependencies =="
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+end

data/integration/apps/rack/bin/test

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  puts "Run: #{args.join(' ')}"
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  puts "\n== Performing setup =="
+  system!('./bin/setup')
+
+  if ENV['TEST_INTEGRATION']
+    puts "\n== Wait for healthy HTTP server... =="
+    system!('bash /vendor/dd-demo/http-health-check')
+  end
+
+  puts "\n== Run test suite =="
+  system!('rspec')
+end

data/integration/apps/rack/config.ru

@@ -0,0 +1,6 @@
+require 'datadog/demo_env'
+require_relative 'app/datadog'
+require_relative 'app/acme'
+
+use Datadog::Contrib::Rack::TraceMiddleware if Datadog::DemoEnv.feature?('tracing')
+run Acme::Application.new

data/integration/apps/rack/config/puma.rb

@@ -0,0 +1,14 @@
+require 'datadog/demo_env'
+
+Datadog::DemoEnv.print_env('Puma master environment')
+
+workers Integer(ENV["WEB_CONCURRENCY"] || 1)
+threads 2, Integer(ENV['WEB_MAX_THREADS'] || 24)
+
+preload_app!
+
+bind 'tcp://0.0.0.0:80'
+
+on_worker_boot do
+  Datadog::DemoEnv.print_env('Puma worker environment')
+end

data/integration/apps/rack/config/unicorn.rb

@@ -0,0 +1,23 @@
+require 'datadog/demo_env'
+
+# config/unicorn.rb
+worker_processes Integer(ENV["WEB_CONCURRENCY"] || 3)
+timeout 15
+preload_app true
+
+Datadog::DemoEnv.print_env('Unicorn master environment')
+
+before_fork do |server, worker|
+  Signal.trap 'TERM' do
+    puts 'Unicorn master intercepting TERM and sending myself QUIT instead'
+    Process.kill 'QUIT', Process.pid
+  end
+end
+
+after_fork do |server, worker|
+  Signal.trap 'TERM' do
+    puts 'Unicorn worker intercepting TERM and doing nothing. Wait for master to send QUIT'
+  end
+
+  Datadog::DemoEnv.print_env('Unicorn worker environment')
+end