oboe 2.2.6 → 2.3.2

data/lib/oboe/inst/rack.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 by Tracelytics, Inc.
+# Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
 module Oboe
@@ -9,20 +9,17 @@ module Oboe
       @app = app
     end
 
-    def call(env)
+    def collect(env)
       report_kvs = {}
-      xtrace = nil
 
       begin
-        xtrace = env['HTTP_X_TRACE'] if env.is_a?(Hash)
-        
         req = ::Rack::Request.new(env)
         report_kvs[:SampleRate]        = Oboe::Config[:sample_rate]
         report_kvs[:SampleSource]      = Oboe::Config[:sample_source]
         report_kvs['HTTP-Host']        = req.host
         report_kvs['Port']             = req.port
         report_kvs['Proto']            = req.scheme
-        report_kvs['Query-String']     = req.query_string unless req.query_string.blank?
+        report_kvs['Query-String']     = req.query_string unless req.query_string.empty?
         report_kvs[:URL]               = req.path
         report_kvs[:Method]            = req.request_method
         report_kvs['AJAX']             = true if req.xhr?
@@ -43,11 +40,21 @@ module Oboe
         # Discard any potential exceptions. Debug log and report whatever we can.
         Oboe.logger.debug "[oboe/debug] Rack KV collection error: #{e.inspect}"
       end
+      report_kvs
+    end
+
+    def call(env)
+      report_kvs = {}
+      xtrace = env.is_a?(Hash) ? env['HTTP_X_TRACE'] : nil
 
       result, xtrace = Oboe::API.start_trace('rack', xtrace, report_kvs) do
 
         status, headers, response = @app.call(env)
-        Oboe::API.log(nil, 'info', { :Status => status })
+
+        if Oboe.tracing?
+          report_kvs = collect(env) 
+          Oboe::API.log(nil, 'info', report_kvs.merge!({ :Status => status }))
+        end
 
         [status, headers, response]
       end

data/lib/oboe/inst/resque.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 by Tracelytics, Inc.
+# Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
 require 'socket'
@@ -29,7 +29,7 @@ module Oboe
             end
           end
           
-          report_kvs[:Backtrace] = Oboe::API.backtrace
+          report_kvs[:Backtrace] = Oboe::API.backtrace if Oboe::Config[:resque][:collect_backtraces]
         rescue
         end
 

data/lib/oboe/instrumentation.rb

@@ -1,3 +1,6 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
 module Oboe
   module Inst
     def self.load_instrumentation

data/lib/oboe/loading.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 by Tracelytics, Inc.
+# Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
 require 'digest/sha1'
@@ -78,8 +78,6 @@ module Oboe
     # Load the oboe tracing API
     # 
     def self.require_api
-      require 'oboe/version'
-
       pattern = File.join(File.dirname(__FILE__), 'api', '*.rb')
       Dir.glob(pattern) do |f|
         require f
@@ -91,8 +89,6 @@ module Oboe
       rescue LoadError => e
         Oboe.logger.fatal "[oboe/error] Couldn't load oboe api."
       end
-      
-      require 'oboe/config'
     end
 
     ## 
@@ -114,5 +110,5 @@ end
 
 Oboe::Loading.require_api
 Oboe::Loading.load_framework_instrumentation
-Oboe::API.report_init('ruby')
+Oboe::Reporter.start
 

data/lib/oboe/logger.rb

@@ -1,6 +1,8 @@
-# Copyright (c) 2013 by AppNeta
+# Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
+require 'logger'
+
 module Oboe
   class << self
     attr_accessor :logger

data/lib/oboe/ruby.rb

@@ -1,3 +1,6 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
 module Oboe
   module Ruby
     def self.initialize

data/lib/oboe/util.rb

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
 
 module Oboe
   module Util

data/lib/oboe/version.rb

@@ -1,8 +1,11 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
 module Oboe
   module Version
     MAJOR = 2
-    MINOR = 2
-    PATCH = 6
+    MINOR = 3
+    PATCH = 2
     BUILD = nil
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')

data/lib/oboe_fu.rb

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
 
 puts "[oboe_fu] This gem has been deprecated.  Please use the 'oboe' gem instead."
 require 'oboe'

data/lib/oboe_metal.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 by Tracelytics, Inc.
+# Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
 module Oboe_metal
@@ -12,7 +12,7 @@ module Oboe_metal
     class << self
       attr_accessor :layer_op
 
-      def log(layer, label, options = {}, with_backtrace = true)
+      def log(layer, label, options = {}, with_backtrace = false)
         evt = Oboe::Context.createEvent()
         evt.addInfo("Layer", layer.to_s)
         evt.addInfo("Label", label.to_s)
@@ -37,6 +37,27 @@ module Oboe_metal
   end
   
   module Reporter
+    ##
+    # Initialize the Oboe Context, reporter and report the initialization
+    #
+    def self.start
+      begin
+        Oboe_metal::Context.init() 
+
+        if ENV['RACK_ENV'] == "test"
+          Oboe.reporter = Oboe::FileReporter.new("./tmp/trace_output.bson")
+        else
+          Oboe.reporter = Oboe::UdpReporter.new(Oboe::Config[:reporter_host])
+        end
+
+        Oboe::API.report_init('ruby') unless ["development", "test"].include? ENV['RACK_ENV']
+      
+      rescue Exception => e
+        $stderr.puts e.message
+        raise
+      end
+    end
+    
     def self.sendReport(evt)
       Oboe.reporter.sendReport(evt)
     end
@@ -83,12 +104,4 @@ module Oboe
   end
 end
 
-begin
-  Oboe_metal::Context.init() 
-  Oboe.reporter = Oboe::UdpReporter.new("127.0.0.1")
-
-rescue Exception => e
-  $stderr.puts e.message
-  raise
-end
 

data/lib/rails/generators/oboe/install_generator.rb

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
 
 module Oboe
   class InstallGenerator < ::Rails::Generators::Base
@@ -7,18 +9,18 @@ module Oboe
     def copy_initializer
       # Set defaults
       @tracing_mode = 'through'
-      @sampling_rate = '300000'
+      @sample_rate = '300000'
       @verbose = 'false'
 
       say ""
       say shell.set_color "Welcome to the TraceView Ruby instrumentation setup.", :green, :bold
       say ""
-      say "To instrument your Rails application, you have the option to setup sampling strategies here."
+      say "To instrument your Rails application, you can setup your tracing strategy here."
       say ""
       say shell.set_color "Documentation Links", :magenta
       say "-------------------"
       say ""
-      say "Details on configuring your sampling rate:"
+      say "Details on configuring your sample rate:"
       say "http://support.tv.appneta.com/support/solutions/articles/86336-configuring-sampling"
       say ""
       say "More information on instrumenting Ruby applications can be found here:"

data/lib/rails/generators/oboe/templates/oboe_initializer.rb

@@ -1,7 +1,7 @@
 # AppNeta TraceView Initializer (the oboe gem)
 # http://www.appneta.com/products/traceview/
 #
-# Details on configuring your sampling rate:
+# Details on configuring your sample rate:
 # http://support.tv.appneta.com/support/solutions/articles/86336
 #
 # More information on instrumenting Ruby applications can be found here:
@@ -16,7 +16,7 @@ if defined?(Oboe::Config)
   Oboe::Config[:tracing_mode] = '<%= @tracing_mode %>'
   
   # sample_rate is a value from 0 - 1m indicating the fraction of requests per million to trace
-  # Oboe::Config[:sample_rate] = <%= @sampling_rate %>
+  # Oboe::Config[:sample_rate] = <%= @sample_rate %>
   
   # Verbose output of instrumentation initialization
   # Oboe::Config[:verbose] = <%= @verbose %>
@@ -56,5 +56,27 @@ if defined?(Oboe::Config)
   # Oboe::Config[:memcached][:enabled] = true
   # Oboe::Config[:mongo][:enabled] = true
   # Oboe::Config[:moped][:enabled] = true
+  # Oboe::Config[:nethttp][:enabled] = true
   # Oboe::Config[:resque][:enabled] = true
+  
+  #
+  # Enabling/Disabling Backtrace Collection
+  #
+  # Instrumentation can optionally collect backtraces as they collect
+  # performance metrics.  Note that this has a negative impact on
+  # performance but can be useful when trying to locate the source of
+  # a certain call or operation.  
+  #
+  # Oboe::Config[:action_controller][:collect_backtraces] = true
+  # Oboe::Config[:active_record][:collect_backtraces] = true
+  # Oboe::Config[:action_view][:collect_backtraces] = true
+  # Oboe::Config[:cassandra][:collect_backtraces] = true
+  # Oboe::Config[:dalli][:collect_backtraces] = false
+  # Oboe::Config[:memcache][:collect_backtraces] = false
+  # Oboe::Config[:memcached][:collect_backtraces] = false
+  # Oboe::Config[:mongo][:collect_backtraces] = true
+  # Oboe::Config[:moped][:collect_backtraces] = true
+  # Oboe::Config[:nethttp][:collect_backtraces] = true
+  # Oboe::Config[:resque][:collect_backtraces] = true
+  #
 end

data/oboe.gemspec

@@ -2,18 +2,24 @@ $:.push File.expand_path("../lib", __FILE__)
 require "oboe/version"
 
 Gem::Specification.new do |s|
-    s.name = %q{oboe}
-    s.version = Oboe::Version::STRING
-    s.date = Time.now.strftime('%Y-%m-%d')
-    s.authors = ["Tracelytics, Inc."]
-    s.email = %q{contact@tracelytics.com}
-    s.summary = %q{Tracelytics instrumentation gem}
-    s.homepage = %q{http://tracelytics.com}
-    s.description = %q{The oboe gem provides AppNeta instrumentation for Ruby and Ruby frameworks.}
-    s.extra_rdoc_files = ["LICENSE"]
-    s.files = `git ls-files`.split("\n")
-    s.extensions = ['ext/oboe_metal/extconf.rb']
-    s.test_files  = Dir.glob("{spec}/**/*.rb")
-    s.add_development_dependency 'rake'
-    s.add_development_dependency 'rspec'
+  s.name = %q{oboe}
+  s.version = Oboe::Version::STRING
+  s.date = Time.now.strftime('%Y-%m-%d')
+
+  s.authors = ["Peter Giacomo Lombardo", "Spiros Eliopoulos"]
+  s.email = %q{traceviewsupport@appneta.com}
+  s.homepage = %q{http://www.appneta.com/application-performance-management}
+  s.summary = %q{AppNeta TraceView performance instrumentation gem for Ruby}
+  s.description = %q{The oboe gem provides TraceView instrumentation for Ruby and Ruby frameworks.}
+
+  s.extra_rdoc_files = ["LICENSE"]
+  s.files = `git ls-files`.split("\n")
+  s.test_files  = Dir.glob("{test}/**/*.rb")
+    
+  s.extensions = ['ext/oboe_metal/extconf.rb']
+   
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'bson'
 end
+

data/test/instrumentation/cassandra_test.rb

@@ -0,0 +1,331 @@
+require 'minitest_helper'
+
+describe Oboe::Inst::Cassandra do
+  before do
+    clear_all_traces 
+
+    @client = Cassandra.new('TRCassInstr', '127.0.0.1:9160',
+                           :retries => 2, :connect_timeout => 1,
+                           :timeout => 5)
+    @client.disable_node_auto_discovery!
+
+    # These are standard entry/exit KVs that are passed up with all mongo operations
+    @entry_kvs = {
+      'Layer' => 'cassandra',
+      'Label' => 'entry',
+      'RemoteHost' => '127.0.0.1',
+      'RemotePort' => '9160' }
+
+    @exit_kvs = { 'Layer' => 'cassandra', 'Label' => 'exit' }
+    @collect_backtraces = Oboe::Config[:cassandra][:collect_backtraces]
+  end
+
+  after do
+    Oboe::Config[:cassandra][:collect_backtraces] = @collect_backtraces
+    @client.disconnect!
+  end
+
+  it 'Stock Cassandra should be loaded, defined and ready' do
+    defined?(::Cassandra).wont_match nil 
+  end
+
+  it 'Cassandra should have oboe methods defined' do
+    [ :insert, :remove, :count_columns, :get_columns, :multi_get_columns, :get,
+      :multi_get, :get_range_single, :get_range_batch, :get_indexed_slices,
+      :create_index, :drop_index, :add_column_family, :drop_column_family,
+      :add_keyspace, :drop_keyspace ].each do |m|
+      ::Cassandra.method_defined?("#{m}_with_oboe").must_equal true
+    end
+    # Special 'exists?' case
+    ::Cassandra.method_defined?("exists_with_oboe?").must_equal true
+  end
+  
+  it 'should trace insert' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      user = {'screen_name' => 'larry', "blah" => "ok"}
+      @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "insert"
+    traces[1]['Cf'].must_equal "Users"
+    traces[1]['Key'].must_equal "5"
+    traces[1]['Consistency'].must_equal "1"
+    traces[1]['Ttl'].must_equal "600"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+
+  it 'should trace remove' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.remove(:Users, '5', 'blah')
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "remove"
+    traces[1]['Cf'].must_equal "Users"
+    traces[1]['Key'].must_equal "5"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+  
+  it 'should trace count_columns' do
+    @client.insert(:Statuses, '12', {'body' => 'v1', 'user' => 'v2'})
+    
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.count_columns(:Statuses, '12', :count => 50)
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "count_columns"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1]['Key'].must_equal "12"
+    traces[1]['Count'].must_equal "50"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+
+  it 'should trace get_columns' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.get_columns(:Statuses, '12', ['body'])
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "get_columns"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1]['Key'].must_equal "12"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+
+  it 'should trace multi_get_columns' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.multi_get_columns(:Users, ['12', '5'], ['body'])
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "multi_get_columns"
+    traces[1]['Cf'].must_equal "Users"
+    traces[1]['Key'].must_equal "[\"12\", \"5\"]"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+
+  it 'should trace get' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.get(:Statuses, '12', :reversed => true)
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "get"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1]['Key'].must_equal "12"
+    traces[1]['Reversed'].must_equal "true"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+
+  it 'should trace exists?' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.exists?(:Statuses, '12')
+      @client.exists?(:Statuses, '12', 'body')
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 6
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "exists?"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1]['Key'].must_equal "12"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+    
+    traces[3]['Op'].must_equal "exists?"
+    traces[3]['Cf'].must_equal "Statuses"
+    traces[3]['Key'].must_equal "12"
+    traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+  end
+
+  it 'should trace get_range_keys' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.get_range_keys(:Statuses, :key_count => 4)
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "get_range_batch"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+
+  it 'should trace create_index and drop_index' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.create_index('TRCassInstr', 'Statuses', 'x', 'LongType')
+      @client.drop_index('TRCassInstr', 'Statuses', 'x')
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 6
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "create_index"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1]['Keyspace'].must_equal "TRCassInstr"
+    traces[1]['Column_name'].must_equal "x"
+    traces[1]['Validation_class'].must_equal "LongType"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+    
+    validate_event_keys(traces[3], @entry_kvs)
+    traces[3]['Op'].must_equal "drop_index"
+    traces[3]['Cf'].must_equal "Statuses"
+    traces[3]['Keyspace'].must_equal "TRCassInstr"
+    traces[3]['Column_name'].must_equal "x"
+    traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[4], @exit_kvs)
+  end
+
+  it 'should trace get_indexed_slices' do
+    @client.create_index('TRCassInstr', 'Statuses', 'x', 'LongType')
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      expressions   =  [
+                         { :column_name => 'x',
+                           :value => [0,20].pack("NN"),
+                           :comparison => "=="},
+                         { :column_name => 'non_indexed',
+                           :value => [5].pack("N*"),
+                           :comparison => ">"} ]
+      @client.get_indexed_slices(:Statuses, expressions).length
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "get_indexed_slices"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+  end
+  
+  it 'should trace add and remove of column family' do
+    cf_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
+    cf_def = CassandraThrift::CfDef.new(:keyspace => "TRCassInstr", :name => cf_name)
+
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.add_column_family(cf_def)
+      @client.drop_column_family(cf_name)
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 6
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "add_column_family"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+    
+    traces[3]['Op'].must_equal "drop_column_family"
+    traces[3]['Cf'].must_equal cf_name
+    traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+  end
+  
+  it 'should trace add and remove of keyspace' do
+    ks_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
+    column_families = [{:name =>"a"}, {:name => "b", :type => :super}]
+    ks_def = CassandraThrift::KsDef.new(:name => ks_name,
+              :strategy_class => "org.apache.cassandra.locator.SimpleStrategy",
+              :replication_factor => 1,
+              :cf_defs => [])
+
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.add_keyspace(ks_def)
+      @client.keyspace = ks_name
+      @client.drop_keyspace(ks_name)
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 6
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "add_keyspace"
+    traces[1]['Name'].must_equal ks_name
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
+    
+    traces[3]['Op'].must_equal "drop_keyspace"
+    traces[3]['Name'].must_equal ks_name
+    traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+  end
+  
+  it "should obey :collect_backtraces setting when true" do
+    Oboe::Config[:cassandra][:collect_backtraces] = true
+
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      user = {'screen_name' => 'larry', "blah" => "ok"}
+      @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+    end
+
+    traces = get_all_traces
+    layer_has_key(traces, 'cassandra', 'Backtrace')
+  end
+
+  it "should obey :collect_backtraces setting when false" do
+    Oboe::Config[:cassandra][:collect_backtraces] = false
+
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      user = {'screen_name' => 'larry', "blah" => "ok"}
+      @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+    end
+
+    traces = get_all_traces
+    layer_doesnt_have_key(traces, 'cassandra', 'Backtrace')
+  end
+  
+end