traceview 3.0.0-java

data/Rakefile

@@ -0,0 +1,109 @@
+#!/usr/bin/env rake
+
+require 'rubygems'
+require 'bundler/setup'
+require 'rake/testtask'
+require 'appraisal'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+  t.ruby_opts = []
+  # t.ruby_opts << ['-w']
+  if defined?(JRUBY_VERSION)
+    t.ruby_opts << ["-J-javaagent:/usr/local/tracelytics/tracelyticsagent.jar"]
+  end
+end
+
+if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"]
+  task :default => :appraisal
+else
+  task :default => :test
+end
+
+desc "Build the gem's c extension"
+task :compile do
+  unless defined?(JRUBY_VERSION)
+    puts "== Building the c extension against Ruby #{RUBY_VERSION}"
+
+    pwd     = Dir.pwd
+    ext_dir = File.expand_path('ext/oboe_metal')
+    lib_dir = File.expand_path('lib')
+    symlink = File.expand_path('lib/oboe_metal.so')
+    so_file = File.expand_path('ext/oboe_metal/oboe_metal.so')
+
+    Dir.chdir ext_dir
+    cmd = [ Gem.ruby, 'extconf.rb']
+    sh cmd.join(' ')
+    sh '/usr/bin/env make'
+    File.delete symlink if File.exist? symlink
+
+    if File.exists? so_file
+      File.symlink so_file, symlink
+      Dir.chdir pwd
+      puts "== Extension built and symlink'd to #{symlink}"
+    else
+      Dir.chdir pwd
+      puts "!! Extension failed to build (see above).  Are the base TraceView packages installed?"
+      puts "!! See https://support.appneta.com/cloud/installing-traceview"
+    end
+  else
+    puts "== Nothing to do under JRuby."
+  end
+end
+
+desc "Clean up extension build files"
+task :clean do
+  unless defined?(JRUBY_VERSION)
+    pwd     = Dir.pwd
+    ext_dir = File.expand_path('ext/oboe_metal')
+    lib_dir = File.expand_path('lib')
+    symlink = File.expand_path('lib/oboe_metal.so')
+    so_file = File.expand_path('ext/oboe_metal/oboe_metal.so')
+
+    Dir.chdir ext_dir
+    sh '/usr/bin/env make clean'
+
+    Dir.chdir pwd
+  else
+    puts "== Nothing to do under JRuby."
+  end
+end
+
+desc "Remove all built files and extensions"
+task :distclean do
+  unless defined?(JRUBY_VERSION)
+    pwd     = Dir.pwd
+    ext_dir = File.expand_path('ext/oboe_metal')
+    lib_dir = File.expand_path('lib')
+    symlink = File.expand_path('lib/oboe_metal.so')
+    so_file = File.expand_path('ext/oboe_metal/oboe_metal.so')
+    mkmf_log = File.expand_path('ext/oboe_metal/mkmf.log')
+
+    if File.exists? mkmf_log
+      Dir.chdir ext_dir
+      File.delete symlink if File.exist? symlink
+      sh '/usr/bin/env make distclean'
+
+      Dir.chdir pwd
+    else
+      puts "Nothing to distclean. (nothing built yet?)"
+    end
+  else
+    puts "== Nothing to do under JRuby."
+  end
+end
+
+desc "Rebuild the gem's c extension"
+task :recompile => [ :distclean, :compile ]
+
+task :console do
+  ENV['TRACEVIEW_GEM_VERBOSE'] = 'true'
+  Bundler.require(:default, :development)
+  TraceView::Config[:tracing_mode] = :always
+  ARGV.clear
+  Pry.start
+end
+
+

data/examples/DNT.md

@@ -0,0 +1,35 @@
+By default, the TraceView oboe gem will not trace routes with extensions
+for common static files.  Examples of such files may be images,
+javascript, pdfs and text files.
+
+This is done by using the regular expression stored in
+`Oboe::Config[:dnt_regexp]`:
+
+    .(jpg|jpeg|gif|png|ico|css|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|js|flv|swf|ttf|woff|svg|less)$
+
+This string is used as a regular expression and is tested against
+candidate URLs to be instrumented.
+
+To replace the pattern in use, you can update this regular expression
+string.  Here are some examples.
+
+If you prefer that you want your javascript and CSS files instrumented,
+you can update `Oboe::Config[:dnt_regexp]` with an updated regexp
+pattern (without the "js" and "css" entries):
+
+    .(jpg|jpeg|gif|png|ico|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|flv|swf|ttf|woff|svg|less)$
+
+If you prefer to not instrument all javascript files except for one
+named `show.js`, you could put this assignment into your initializer,
+rackup file or application.rb (note that this example uses a standard
+regexp [negative
+look-behind](http://www.regular-expressions.info/lookaround.html) that
+isn't supported in Ruby 1.8):
+
+    Oboe::Config[:dnt_regexp] = "(\.js$)(?<!show.js)"
+
+Since this pattern is used with the standard Ruby Regexp class, you can
+use any Regexp supported pattern.  See the documentation on Ruby Regexp
+[here](https://www.omniref.com/ruby/2.2.0/symbols/Regexp?d=380181456&n=0#doc_uncollapsed=true&d=380181456&n=0)
+or you can also view the oboe gem [source code documentation for this
+feature](https://github.com/appneta/oboe-ruby/blob/master/lib/oboe/config.rb#L74).

data/examples/carrying_context.rb

@@ -0,0 +1,225 @@
+###############################################################
+# A brief overview of TraceView tracing context
+###############################################################
+#
+# Tracing context is the state held when TraceView is instrumenting a
+# transaction, block, request etc..  This context is advanced as
+# new blocks are instrumented and this chain of context is used
+# by TraceView to later reassemble performance data to be displayed
+# in the TraceView dashboard.
+#
+# Tracing context is non-existent until established by calling
+# `Oboe::API.start_trace` or `Oboe::API.log_start`.  Those methods
+# are part of the high-level and low-level API respectively.
+#
+# After a tracing context is established, that context can be
+# continued by calling `Oboe::API.trace` or `Oboe::API.log_entry`.
+# These methods will advance an existing context but not start
+# new one.
+#
+# For example, when a web request comes into a stack, a tracing
+# context is established using `Oboe::API.log_start` as the request
+# enters through the rack middleware via `::Oboe::Rack`.
+#
+# That tracing context is then continued using `Oboe::API.trace` or
+# `Oboe::API.log_entry` for each subsequent layer such as Rails,
+# ActiveRecord, Redis, Memcache, ActionView, Mongo (etc...) until
+# finally request processing is complete and the tracing context
+# is cleared (Oboe::Context.clear)
+#
+
+###############################################################
+# Carrying Context
+###############################################################
+#
+# The tracing context exists in the form of an X-Trace string and
+# can be retrieved using 'Oboe::Context.toString'
+#
+# xtrace = Oboe::Context.toString
+#
+# => "1B4EDAB9E028CA3C81BCD57CC4644B4C4AE239C7B713F0BCB9FAD6D562"
+#
+# Tracing context can also be picked up from a pre-existing
+# X-Trace string:
+#
+# xtrace = "1B4EDAB9E028CA3C81BCD57CC4644B4C4AE239C7B713F0BCB9FAD6D562"
+#
+# Oboe::Context.fromString(xtrace)
+#
+# With these two methods, context can be passed across threads,
+# processes (via fork) and in requests (such as external HTTP
+# requests where the X-Trace is inserted in request headers).
+#
+#
+
+###############################################################
+# Two Options for Spawned Tracing
+###############################################################
+#
+# When your application needs to instrument code that forks,
+# spawns a thread or does something in-parallel, you have the
+# option to either link those child traces to the parent or
+# trace them as individuals (but with identifying information).
+#
+# Linking parent and child has it's benefits as in the
+# TraceView dashboard, you will see how a process may spawn
+# a task in parallel and in a single view see the performance
+# of both.
+#
+# The limitation of this is that this is only useful if your
+# parent process spawns only a limited number of child traces.
+#
+# If your parent process is spawning many child tasks (e.g.
+# twenty, hundreds, thousands or more) it's best to trace those
+# child tasks as individuals and pass in identifier Key-Values
+# (such as task ID, job ID etc..)
+#
+# In the examples below, I show implementations of both linked
+# asynchronous traces and separated independent traces.
+
+###############################################################
+# Thread - with separated traces
+###############################################################
+
+Oboe::API.log_start('parent')
+
+# Get the work to be done
+job = get_work
+
+Thread.new do
+  # This is a new thread so there is no pre-existing context so
+  # we'll call `Oboe::API.log_start` to start a new trace context.
+  Oboe::API.log_start('worker_thread', { :job_id => job.id })
+
+  # Do the work
+  do_the_work(job)
+
+  Oboe::API.log_end('worker_thread')
+end
+
+Oboe::API.log_end('parent')
+
+
+
+###############################################################
+#
+# This will generate two independent traces with the following
+# topology.
+#
+# 'parent'
+# ------------------------------------------------------------
+#
+# 'worker_thread'
+# ------------------------------------------------------------
+#
+
+###############################################################
+# Thread - with linked asynchronous traces
+###############################################################
+
+# Since the following example spawns a thread without waiting
+# for it to return, we carry over the context and we mark the
+# trace generated in that thread to be asynchronous using
+# the `Async` flag.
+
+Oboe::API.log_start('parent')
+
+# Save the context to be imported in spawned thread
+tracing_context = Oboe::Context.toString
+
+# Get the work to be done
+job = get_work
+
+Thread.new do
+  # Restore context
+  Oboe::Context.fromString(tracing_context)
+
+  Oboe::API.log_entry('worker_thread')
+
+  # Do the work
+  do_the_work(job)
+
+  Oboe::API.log_exit('worker_thread', { 'Async' => 1 })
+end
+
+Oboe::API.log_end('parent')
+
+###############################################################
+#
+# This will generate a single trace with an asynchronous
+# branch like the following
+#
+# 'parent'
+# ------------------------------------------------------------
+#     \
+#      \
+#       ------------------------------------------------------
+#        'worker_thread'
+#
+
+###############################################################
+# Process via fork - with separated traces
+###############################################################
+
+Oboe::API.start_trace('parent_process') do
+
+  # Get some work to process
+  job = get_job
+
+  # fork process to handle work
+  fork do
+    # Since fork does a complete process copy, the tracing_context still exists
+    # so we have to clear it and start again.
+    Oboe::Context.clear
+
+    Oboe::API.start_trace('worker_process', nil, { :job_id => job.id }) do
+      do_work(job)
+    end
+  end
+
+end
+
+###############################################################
+#
+# This will generate two independent traces:
+#
+# 'parent_process'
+# ------------------------------------------------------------
+#
+# 'worker_process'
+# ------------------------------------------------------------
+#
+###############################################################
+# Process via fork - with linked asynchronous traces
+###############################################################
+
+Oboe::API.start_trace('parent_process') do
+
+  # Get some work to process
+  job = get_job
+
+  # fork process to handle work
+  fork do
+    # Since fork does a complete process copy, the tracing_context still exists
+    # although we'll have to mark these traces as asynchronous to denote
+    # that it has split off from the main program flow
+
+    Oboe::API.trace('worker_process', { 'Async' => 1 }) do
+      do_work(job)
+    end
+  end
+
+end
+
+###############################################################
+#
+# This will generate a single trace with an asynchronous
+# branch like the following
+#
+# 'parent_process'
+# ------------------------------------------------------------
+#     \
+#      \
+#       ------------------------------------------------------
+#        'worker_process'
+#

data/examples/instrumenting_metal_controller.rb

@@ -0,0 +1,8 @@
+class MetalController < ActionController::Metal
+  def index
+    self.response_body = 'Hello Metal!'
+  end
+
+  include OboeMethodProfiling
+  profile_method :index, 'metal-index'
+end

data/examples/puma_on_heroku_config.rb

@@ -0,0 +1,17 @@
+workers Integer(ENV['WEB_CONCURRENCY'] || 2)
+threads_count = Integer(ENV['MAX_THREADS'] || 5)
+threads threads_count, threads_count
+
+preload_app!
+
+rackup      DefaultRackup
+port        ENV['PORT']     || 3000
+environment ENV['RACK_ENV'] || 'development'
+
+on_worker_boot do
+  ::Oboe.reconnect! if defined?(::Oboe)
+end
+
+on_worker_shutdown do
+  ::Oboe.disconnect! if defined?(::Oboe)
+end

data/examples/tracing_async_threads.rb

@@ -0,0 +1,125 @@
+#
+# This sample demonstrates how to instrument a main loop that
+# retrieves work and spawn threads that do the actual work
+#
+
+require 'math'
+require 'oboe'
+
+Oboe::Config[:tracing_mode] = :always
+Oboe::Config[:verbose] = true
+
+# The parent process/loop which collects data
+while true do
+
+  # For each loop, we instrument the work retrieval.  These traces
+  # will show up as layer 'get_the_work'.
+  Oboe::API.start_trace('get_the_work') do
+
+    work = get_the_work
+
+    # Loop through work and pass to `do_the_work` method
+    # that spawns a thread each time
+    work.each do |j|
+
+      # In the new Thread block, the Oboe tracing context isn't there
+      # so we carry it over manually and pass it to the `start_trace`
+      # method.
+
+      # In the TraceView dashboard, this will show up as parent traces
+      # (layer 'get_the_work') with child traces (layer 'do_the_work').
+
+      tracing_context = Oboe::Context.toString
+
+      Thread.new do
+        result = nil
+
+        Oboe::API.start_trace('do_the_work', tracing_context, { 'Async' => 1 }) do
+          result = do_the_work(j)
+        end
+
+        result
+      end
+    end
+  end
+end
+
+
+##
+# get_the_work
+#
+# Method to retrieve work to do
+#
+def get_the_work
+  # We'll just return random integers as a
+  # fake work load
+  w = []
+  w << rand(25)
+  w << rand(25)
+  w << rand(25)
+end
+
+##
+# do_the_work
+#
+# The work-horse method
+#
+def do_the_work(job_to_do)
+  i = job_to_do
+  i * Math::PI
+end
+
+####################################################
+# Notes
+####################################################
+
+# The above code generates a trace for each loop of the parent data collection process.
+# Those traces have the layer name of `get_the_work` and will show up in the TraceView
+# dashboard as such.
+#
+# Then as threads are spawned to process individual bits of work, we carry over the
+# `tracing_context` and start a new asynchronous trace using `start_trace`.  (An
+# asynchronous trace is noted by passing the `Async` Hash key with a value of `1`).
+#
+# In the TraceView dashboard, the two traces (parent and child; or one to many) will
+# be linked and displayed together as a single trace.
+
+####################################################
+# Caveats
+####################################################
+
+# If the main loop is retrieving many jobs (work) to process on each loop then
+# linking the traces may not be the best strategy as such large relationships
+# are difficult to display correctly in the TraceView dashboard and provide little
+# added value.
+#
+# If there are more than 8 - 12 threads spawned from each loop, then you may want to consider
+# NOT carrying over tracing context into the spawned threads.
+#
+# In this case, you can simply omit `tracing_context` and passing it to `start_trace` in
+# the `Thread.new` block. (lines 32 + 37).  Also remove the `{ Async => 1 }` Hash!
+#
+# This will produce two sets of traces with two the layer names 'get_the_work' +
+# 'do_the_work'.
+#
+# In the TraceView dashboard, you can then separate or unify these traces into
+# independent applications.  e.g. job processor, data retrieval, thread worker etc...
+#
+# An implementation of the work loop without carrying over tracing context would look
+# like the following:
+#
+#    work.each do |j|
+#      Thread.new do
+#        result = nil
+#
+#        Oboe::API.start_trace('do_the_work') do
+#          result = do_the_work(j)
+#        end
+#
+#        result
+#      end
+#    end
+#
+# If anything isn't clear, please don't hesitate to reach us at support (traceviewsupport@appneta.com)
+# or on IRC #appneta @ freenode.
+#

data/examples/tracing_background_jobs.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler.require
+
+# Make sure oboe is at the bottom of your Gemfile.
+# This is likely redundant but just in case.
+require 'oboe'
+
+# Tracing mode can be 'never', 'through' (to follow upstream) or 'always'
+Oboe::Config[:tracing_mode] = 'always'
+
+#
+# Update April 9, 2015 - this is done automagically now
+# and doesn't have to be called manually
+#
+# Load library instrumentation to auto-capture stuff we know about...
+# e.g. ActiveRecord, Cassandra, Dalli, Redis, memcache, mongo
+# Oboe::Ruby.load
+
+# Some KVs to report to the dashboard
+report_kvs = {}
+report_kvs[:command_line_params] = ARGV.to_s
+report_kvs[:user_id] = `whoami`
+
+Oboe::API.start_trace('my_background_job', nil, report_kvs ) do
+  #
+  # Initialization code
+  #
+
+  tasks = get_all_tasks
+
+  tasks.each do |t|
+    # Optional: Here we embed another 'trace' to separate actual
+    # work for each task.  In the TV dashboard, this will show
+    # up as a large 'my_background_job' parent layer with many
+    # child 'task" layers.
+    Oboe::API.trace('task', { :task_id => t.id }) do
+      t.perform
+    end
+  end
+  #
+  # cleanup code
+  #
+end
+
+# Note that we use 'start_trace' in the outer block and 'trace' for
+# any sub-blocks of code we wish to instrument.  The arguments for
+# both methods vary slightly.  Details in RubyDoc:
+# https://www.omniref.com/ruby/gems/oboe/2.7.10.1/symbols/Oboe::API::Tracing#tab=Methods
+
+

data/examples/tracing_forked_processes.rb

@@ -0,0 +1,100 @@
+#
+# This sample demonstrates how to instrument a main loop that
+# retrieves work and calls fork to do the actual work
+#
+
+require 'math'
+require 'oboe'
+
+Oboe::Config[:tracing_mode] = :always
+Oboe::Config[:verbose] = true
+
+# The parent process/loop which collects data
+while true do
+
+  # For each loop, we instrument the work retrieval.  These traces
+  # will show up as layer 'get_the_work'.
+  Oboe::API.start_trace('get_the_work') do
+
+    work = get_the_work
+
+    # Loop through work and pass to `do_the_work` method
+    # that spawns a thread each time
+    work.each do |job|
+      fork do
+        # Since the context is copied from the parent process, we clear it
+        # and start a new trace via `Oboe::API.start_trace`.
+        Oboe::Context.clear
+        result = nil
+
+        Oboe::API.start_trace('do_the_work', nil, :job_id => job.id) do
+          result = do_the_work(job)
+        end
+
+        result
+      end
+    end
+  end
+end
+
+##
+# get_the_work
+#
+# Method to retrieve work to do
+#
+def get_the_work
+  # We'll just return random integers as a
+  # fake work load
+  w = []
+  w << rand(25)
+  w << rand(25)
+  w << rand(25)
+end
+
+##
+# do_the_work
+#
+# The work-horse method
+#
+def do_the_work(job_to_do)
+  i = job_to_do
+  i * Math::PI
+end
+
+#########################################################################
+# Notes
+#########################################################################
+
+# If your parent process only forks a small number of processes per loop (< 5..10),
+# you may want to mark the child traces as asynchronous and have them directly
+# linked to the parent tracing context.
+#
+# The benefit of this is that instead of having two independent traces (parent
+# and child), you will have a single view of the parent trace showing the
+# spawned child process and it's performance in the TraceView dashboard.
+#
+# To do this:
+#   1. Don't clear the context in the child process
+#   2. Use `Oboe::API.trace` instead
+#   3. Pass the `Async` flag to mark this child as asynchronous
+#
+while true do
+  Oboe::API.start_trace('get_the_work') do
+
+    work = get_the_work
+
+    work.each do |job|
+      fork do
+        result = nil
+        # 1 Don't clear context
+        # 2 Use `Oboe::API.trace` instead
+        # 3 Pass the Async flag
+        Oboe::API.trace('do_the_work', { :job_id => job.id, 'Async' => 1 }) do
+          result = do_the_work(job)
+        end
+
+        result
+      end
+    end
+  end
+end

data/examples/unicorn_on_heroku_config.rb

@@ -0,0 +1,28 @@
+worker_processes Integer(ENV["WEB_CONCURRENCY"] || 3)
+timeout 15
+preload_app true
+
+before_fork do |server, worker|
+  Signal.trap 'TERM' do
+    puts 'Unicorn master intercepting TERM and sending myself QUIT instead'
+    Process.kill 'QUIT', Process.pid
+  end
+
+  defined?(ActiveRecord::Base) and
+    ActiveRecord::Base.connection.disconnect!
+
+  defined?(::Oboe) and
+    ::Oboe.disconnect!
+end
+
+after_fork do |server, worker|
+  Signal.trap 'TERM' do
+    puts 'Unicorn worker intercepting TERM and doing nothing. Wait for master to send QUIT'
+  end
+
+  defined?(ActiveRecord::Base) and
+    ActiveRecord::Base.establish_connection
+
+  defined?(::Oboe) and
+    ::Oboe.reconnect!
+end