appoptics_apm 4.1.2 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 279e04ea7b50365f82a752e34561e98553accfbf
-  data.tar.gz: 6c164062efc035368cec63f422689ae0317b8986
+  metadata.gz: f75eee6a2dc08842b68a7dce1a083c6f122fd462
+  data.tar.gz: 7a70d00b5f659de3cb11c2828a6d9695363cce09
 SHA512:
-  metadata.gz: 58716fc2e035939241428b70b45ce5e090238c71758d72fa9b29e2fbdc11fd34f89e97d5a44d6f769d8a3edf80f6ce4e02aaea6e3b1a4b001778970cf43a81ca
-  data.tar.gz: aacc0674ebe52dda370cec24c6a194a4e57bc0e41dddbae3e9e93b0565350f96d42a41119fe895e4148741bfe7262a7396e1c513242a2e6f4d6303f2f3d0022f
+  metadata.gz: 44f247b3789ab9bbd385324a42eb61025759eec224bc85dd7d3abeee6f3e0d4facc0af8541ab99b0897a8286835cc0ba88f6b1f61fa8a59995b359c54796ab1c
+  data.tar.gz: a38c975e81913efad14a501ec159aeba97825ded8f7ade3188443be33025cc76771c813d37bcf9e47f35bd10ebd0aa5900d6526321f194524ad7ef6d2c0bc560

data/.travis.yml

@@ -5,9 +5,14 @@ cache:
   directories:
   - vendor/bundle
 
+env:
+  - DBTYPE=mysql
+  - DBTYPE=mysql2
+  - DBTYPE=postgresql
+
 rvm:
   - 2.5.0
-  - 2.4.3
+#  - 2.4.3
   - 2.3.6
 #  - jruby-9.0.5.0
 
@@ -23,11 +28,6 @@ gemfile:
   - gemfiles/rails51.gemfile
   - gemfiles/delayed_job.gemfile
 
-env:
-  - DBTYPE=mysql
-  - DBTYPE=mysql2
-  - DBTYPE=postgresql
-
 matrix:
   exclude:
     - rvm: 2.5.0
@@ -103,6 +103,8 @@ before_script:
   - psql -c 'create database travis_ci_test;' -U postgres
   - mysql -e 'create database travis_ci_test;'
   - sleep 10
+  - export APPOPTICS_TOKEN_BUCKET_CAPACITY=1000
+  - export APPOPTICS_TOKEN_BUCKET_RATE=1000
 
 script: "N=1 bundle exec rake test"
 

data/Dockerfile_test

@@ -1,7 +1,7 @@
 FROM ubuntu:16.04
 
 # to use with ./run_tests_docker.rb
-# docker build -f Dockerfile-test -t ruby_appoptics_apm .
+# docker build -f Dockerfile_test -t ruby_appoptics_apm .
 
 # install OS packages
 RUN apt-get update \
@@ -39,7 +39,7 @@ RUN . ~/.profile \
     && rbenv install 2.3.6 \
     && rbenv install 2.4.3 \
     && rbenv install 2.5.0
-    && rbenv install jruby-9.1.16.0
+#    && rbenv install jruby-9.1.16.0
 
 # install swig 3.0.8
 RUN curl -SL http://kent.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz \
@@ -68,3 +68,5 @@ RUN apt-get update && \
 ENV PATH="/root/.rbenv/bin:/root/.rbenv/shims:$PATH"
 ENV RUBY_ENV=test
 ENV DOCKER_PSQL_PASS=docker
+ENV APPOPTICS_TOKEN_BUCKET_CAPACITY=1000
+ENV APPOPTICS_TOKEN_BUCKET_RATE=1000

data/Rakefile

@@ -73,13 +73,23 @@ task :fetch_ext_deps do
   end
 
   # The c-lib version is different from the gem version
-  oboe_version = ENV['OBOE_VERSION'] || '2.0.10' # 'latest'
-  oboe_src_dir = "https://s3-us-west-2.amazonaws.com/rc-files-t2/c-lib/#{oboe_version}"
+  oboe_version = ENV['OBOE_VERSION'] || 'latest'
+  oboe_s3_dir = "https://s3-us-west-2.amazonaws.com/rc-files-t2/c-lib/#{oboe_version}"
   ext_src_dir = File.expand_path('ext/oboe_metal/src')
 
   # VERSION is used by extconf.rb to download the correct liboboe when installing the gem
-  %w(oboe.i oboe.h oboe.hpp oboe_debug.h VERSION).each do |filename|
-    remote_file = File.join(oboe_src_dir, filename)
+  remote_file = File.join(oboe_s3_dir, 'VERSION')
+  local_file = File.join(ext_src_dir, 'VERSION')
+  puts "fetching #{remote_file} to #{local_file}"
+  open(remote_file, 'rb') do |rf|
+    content = rf.read
+    File.open(local_file, 'wb') { |f| f.puts content }
+  end
+
+  # oboe and bson header files
+  FileUtils.mkdir_p(File.join(ext_src_dir, 'bson'))
+  %w(oboe.h oboe.hpp oboe_debug.h oboe.i bson/bson.h bson/platform_hacks.h).each do |filename|
+    remote_file = File.join(oboe_s3_dir, 'include', filename)
     local_file = File.join(ext_src_dir, filename)
 
     puts "fetching #{remote_file} to #{local_file}"
@@ -88,6 +98,7 @@ task :fetch_ext_deps do
       File.open(local_file, 'wb') { |f| f.puts content }
     end
   end
+
   FileUtils.cd(ext_src_dir) do
     system('swig -c++ -ruby -module oboe_metal oboe.i')
     FileUtils.rm('oboe.i')

data/appoptics_apm.gemspec

@@ -15,11 +15,13 @@ Gem::Specification.new do |s|
   s.description = %q{The AppOpticsAPM gem provides performance instrumentation for MRI Ruby and related frameworks.}
 
   s.extra_rdoc_files = ["LICENSE"]
-  s.files = `git ls-files`.split("\n").reject { |f| f.match(%r{^(test|gemfiles|examples)/}) }
+  s.files = `git ls-files`.split("\n").reject { |f| f.match(%r{^(test|gemfiles)/}) }
   s.files += ['ext/oboe_metal/src/oboe.h',
               'ext/oboe_metal/src/oboe.hpp',
               'ext/oboe_metal/src/oboe_debug.h',
               'ext/oboe_metal/src/oboe_wrap.cxx',
+              'ext/oboe_metal/src/bson/bson.h',
+              'ext/oboe_metal/src/bson/platform_hacks.h',
               'ext/oboe_metal/src/VERSION']
 
   # TODO this is commented out util we can actually provide gems for different platforms

data/examples/DNT.md

@@ -0,0 +1,35 @@
+By default, the AppOpticsAPM gem will not trace routes with extensions
+for common static files.  Examples of such files may be images,
+javascript, pdfs and text files.
+
+This is done by using the regular expression stored in
+`AppOpticsAPM::Config[:dnt_regexp]`:
+
+    .(jpg|jpeg|gif|png|ico|css|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|js|flv|swf|ttf|woff|svg|less)$
+
+This string is used as a regular expression and is tested against
+candidate URLs to be instrumented.
+
+To replace the pattern in use, you can update this regular expression
+string.  Here are some examples.
+
+If you prefer that you want your javascript and CSS files instrumented,
+you can update `AppOpticsAPM::Config[:dnt_regexp]` with an updated regexp
+pattern (without the "js" and "css" entries):
+
+    .(jpg|jpeg|gif|png|ico|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|flv|swf|ttf|woff|svg|less)$
+
+If you prefer to not instrument all javascript files except for one
+named `show.js`, you could put this assignment into your initializer,
+rackup file or application.rb (note that this example uses a standard
+regexp [negative
+look-behind](http://www.regular-expressions.info/lookaround.html) that
+isn't supported in Ruby 1.8):
+
+    AppOpticsAPM::Config[:dnt_regexp] = "(\.js$)(?<!show.js)"
+
+Since this pattern is used with the standard Ruby Regexp class, you can
+use any Regexp supported pattern.  See the documentation on Ruby Regexp
+[here](https://www.omniref.com/ruby/2.2.0/symbols/Regexp?d=380181456&n=0#doc_uncollapsed=true&d=380181456&n=0)
+or you can also view the oboe gem [source code documentation for this
+feature](https://github.com/tracelytics/ruby-appoptics/blob/master/lib/appoptics/config.rb#L129).

data/examples/carrying_context.rb

@@ -0,0 +1,220 @@
+###############################################################
+# A brief overview of AppOpticsAPM tracing context
+###############################################################
+#
+# Tracing context is the state held when AppOpticsAPM is instrumenting a
+# transaction, block, request etc..  This context is advanced as
+# new blocks are instrumented and this chain of context is used
+# by AppOpticsAPM to later reassemble performance data to be displayed
+# in the AppOptics dashboard.
+#
+# Tracing context is non-existent until established by calling
+# `AppOpticsAPM::API.start_trace` or `AppOpticsAPM::API.log_start`.  Those methods
+# are part of the high-level and low-level API respectively.
+#
+# After a tracing context is established, that context can be
+# continued by calling `AppOpticsAPM::API.trace` or `AppOpticsAPM::API.log_entry`.
+# These methods will advance an existing context but not start
+# new one.
+#
+# For example, when a web request comes into a stack, a tracing
+# context is established using `AppOpticsAPM::API.log_start` as the request
+# enters through the rack middleware via `::AppOpticsAPM::Rack`.
+#
+# That tracing context is then continued using `AppOpticsAPM::API.trace` or
+# `AppOpticsAPM::API.log_entry` for each subsequent layer such as Rails,
+# ActiveRecord, Redis, Memcache, ActionView, Mongo (etc...) until
+# finally request processing is complete and the tracing context
+# is cleared (AppOpticsAPM::Context.clear)
+#
+
+###############################################################
+# Carrying Context
+###############################################################
+#
+# The tracing context exists in the form of an X-Trace string and
+# can be retrieved using 'AppOpticsAPM::Context.toString'
+#
+# xtrace = AppOpticsAPM::Context.toString
+#
+# => "1B4EDAB9E028CA3C81BCD57CC4644B4C4AE239C7B713F0BCB9FAD6D562"
+#
+# Tracing context can also be picked up from a pre-existing
+# X-Trace string:
+#
+# xtrace = "1B4EDAB9E028CA3C81BCD57CC4644B4C4AE239C7B713F0BCB9FAD6D562"
+#
+# AppOpticsAPM::Context.fromString(xtrace)
+#
+# With these two methods, context can be passed across threads,
+# processes (via fork) and in requests (such as external HTTP
+# requests where the X-Trace is inserted in request headers).
+#
+#
+
+###############################################################
+# Two Options for Spawned Tracing
+###############################################################
+#
+# When your application needs to instrument code that forks,
+# spawns a thread or does something in-parallel, you have the
+# option to either link those child traces to the parent or
+# trace them as individuals (but with identifying information).
+#
+# Linking parent and child has it's benefits as in the
+# AppOptics dashboard, you will see how a process may spawn
+# a task in parallel and in a single view see the performance
+# of both.
+#
+# The limitation of this is that this is only useful if your
+# parent process spawns only a limited number of child traces.
+#
+# If your parent process is spawning many child tasks (e.g.
+# twenty, hundreds, thousands or more) it's best to trace those
+# child tasks as individuals and pass in identifier Key-Values
+# (such as task ID, job ID etc..)
+#
+# In the examples below, I show implementations of both linked
+# asynchronous traces and separated independent traces.
+
+###############################################################
+# Thread - with separated traces
+###############################################################
+
+AppOpticsAPM::API.log_start('parent')
+
+# Get the work to be done
+job = get_work
+
+Thread.new do
+  # This is a new thread so there is no pre-existing context so
+  # we'll call `AppOpticsAPM::API.log_start` to start a new trace context.
+  AppOpticsAPM::API.log_start('worker_thread', :job_id => job.id)
+
+  # Do the work
+  do_the_work(job)
+
+  AppOpticsAPM::API.log_end('worker_thread')
+end
+
+AppOpticsAPM::API.log_end('parent')
+
+###############################################################
+#
+# This will generate two independent traces with the following
+# topology.
+#
+# 'parent'
+# ------------------------------------------------------------
+#
+# 'worker_thread'
+# ------------------------------------------------------------
+#
+
+###############################################################
+# Thread - with linked asynchronous traces
+###############################################################
+
+# Since the following example spawns a thread without waiting
+# for it to return, we carry over the context and we mark the
+# trace generated in that thread to be asynchronous using
+# the `Async` flag.
+
+AppOpticsAPM::API.log_start('parent')
+
+# Save the context to be imported in spawned thread
+tracing_context = AppOpticsAPM::Context.toString
+
+# Get the work to be done
+job = get_work
+
+Thread.new do
+  # Restore context
+  AppOpticsAPM::Context.fromString(tracing_context)
+
+  AppOpticsAPM::API.log_entry('worker_thread')
+
+  # Do the work
+  do_the_work(job)
+
+  AppOpticsAPM::API.log_exit('worker_thread', :Async => 1)
+end
+
+AppOpticsAPM::API.log_end('parent')
+
+###############################################################
+#
+# This will generate a single trace with an asynchronous
+# branch like the following
+#
+# 'parent'
+# ------------------------------------------------------------
+#     \
+#      \
+#       ------------------------------------------------------
+#        'worker_thread'
+#
+
+###############################################################
+# Process via fork - with separated traces
+###############################################################
+
+AppOpticsAPM::API.start_trace('parent_process') do
+  # Get some work to process
+  job = get_job
+
+  # fork process to handle work
+  fork do
+    # Since fork does a complete process copy, the tracing_context still exists
+    # so we have to clear it and start again.
+    AppOpticsAPM::Context.clear
+
+    AppOpticsAPM::API.start_trace('worker_process', nil, :job_id => job.id) do
+      do_work(job)
+    end
+  end
+
+end
+
+###############################################################
+#
+# This will generate two independent traces:
+#
+# 'parent_process'
+# ------------------------------------------------------------
+#
+# 'worker_process'
+# ------------------------------------------------------------
+#
+###############################################################
+# Process via fork - with linked asynchronous traces
+###############################################################
+
+AppOpticsAPM::API.start_trace('parent_process') do
+  # Get some work to process
+  job = get_job
+
+  # fork process to handle work
+  fork do
+    # Since fork does a complete process copy, the tracing_context still exists
+    # although we'll have to mark these traces as asynchronous to denote
+    # that it has split off from the main program flow
+
+    AppOpticsAPM::API.trace('worker_process', :Async => 1) do
+      do_work(job)
+    end
+  end
+end
+
+###############################################################
+#
+# This will generate a single trace with an asynchronous
+# branch like the following
+#
+# 'parent_process'
+# ------------------------------------------------------------
+#     \
+#      \
+#       ------------------------------------------------------
+#        'worker_process'
+#

data/examples/instrumenting_metal_controller.rb

@@ -0,0 +1,8 @@
+class MetalController < ActionController::Metal
+  def index
+    self.response_body = 'Hello Metal!'
+  end
+
+  include AppOpticsAPMMethodProfiling
+  profile_method :index, 'metal-index'
+end

data/examples/puma_on_heroku_config.rb

@@ -0,0 +1,17 @@
+workers Integer(ENV['WEB_CONCURRENCY'] || 2)
+threads_count = Integer(ENV['MAX_THREADS'] || 5)
+threads threads_count, threads_count
+
+preload_app!
+
+rackup      DefaultRackup
+port        ENV['PORT']     || 3000
+environment ENV['RACK_ENV'] || 'development'
+
+on_worker_boot do
+  ::AppOpticsAPM.reconnect! if defined?(::AppOpticsAPM)
+end
+
+on_worker_shutdown do
+  ::AppOpticsAPM.disconnect! if defined?(::AppOpticsAPM)
+end

data/examples/tracing_async_threads.rb

@@ -0,0 +1,124 @@
+#
+# This sample demonstrates how to instrument a main loop that
+# retrieves work and spawn threads that do the actual work
+#
+
+require 'math'
+require 'oboe'
+
+AppOpticsAPM::Config[:tracing_mode] = :always
+AppOpticsAPM::Config[:verbose] = true
+
+# The parent process/loop which collects data
+Kernel.loop do
+
+  # For each loop, we instrument the work retrieval.  These traces
+  # will show up as layer 'get_the_work'.
+  AppOpticsAPM::API.start_trace('get_the_work') do
+    work = get_the_work
+
+    # Loop through work and pass to `do_the_work` method
+    # that spawns a thread each time
+    work.each do |j|
+
+      # In the new Thread block, the AppOpticsAPM tracing context isn't there
+      # so we carry it over manually and pass it to the `start_trace`
+      # method.
+
+      # In the AppOptics dashboard, this will show up as parent traces
+      # (layer 'get_the_work') with child traces (layer 'do_the_work').
+
+      tracing_context = AppOpticsAPM::Context.toString
+
+      Thread.new do
+        result = nil
+
+        AppOpticsAPM::API.start_trace('do_the_work', tracing_context, :Async => 1) do
+          result = do_the_work(j)
+        end
+
+        result
+      end
+    end
+  end
+  sleep 5
+end
+
+
+##
+# get_the_work
+#
+# Method to retrieve work to do
+#
+def get_the_work
+  # We'll just return random integers as a
+  # fake work load
+  w = []
+  w << rand(25)
+  w << rand(25)
+  w << rand(25)
+end
+
+##
+# do_the_work
+#
+# The work-horse method
+#
+def do_the_work(job_to_do)
+  i = job_to_do
+  i * Math::PI
+end
+
+####################################################
+# Notes
+####################################################
+
+# The above code generates a trace for each loop of the parent data collection process.
+# Those traces have the layer name of `get_the_work` and will show up in the AppOptics
+# dashboard as such.
+#
+# Then as threads are spawned to process individual bits of work, we carry over the
+# `tracing_context` and start a new asynchronous trace using `start_trace`.  (An
+# asynchronous trace is noted by passing the `Async` Hash key with a value of `1`).
+#
+# In the AppOptics dashboard, the two traces (parent and child; or one to many) will
+# be linked and displayed together as a single trace.
+
+####################################################
+# Caveats
+####################################################
+
+# If the main loop is retrieving many jobs (work) to process on each loop then
+# linking the traces may not be the best strategy as such large relationships
+# are difficult to display correctly in the AppOptics dashboard and provide little
+# added value.
+#
+# If there are more than 8 - 12 threads spawned from each loop, then you may want to consider
+# NOT carrying over tracing context into the spawned threads.
+#
+# In this case, you can simply omit `tracing_context` and passing it to `start_trace` in
+# the `Thread.new` block. (lines 32 + 37).  Also remove the `{ Async => 1 }` Hash!
+#
+# This will produce two sets of traces with two the layer names 'get_the_work' +
+# 'do_the_work'.
+#
+# In the AppOptics dashboard, you can then separate or unify these traces into
+# independent applications.  e.g. job processor, data retrieval, thread worker etc...
+#
+# An implementation of the work loop without carrying over tracing context would look
+# like the following:
+#
+#    work.each do |j|
+#      Thread.new do
+#        result = nil
+#
+#        AppOpticsAPM::API.start_trace('do_the_work') do
+#          result = do_the_work(j)
+#        end
+#
+#        result
+#      end
+#    end
+#
+# If anything isn't clear, please don't hesitate to reach us at support (support@appoptics.com).
+#