oboe 2.3.2 → 2.3.3.7

data/lib/oboe.rb

@@ -5,16 +5,25 @@ begin
   require 'oboe/version'
   require 'oboe/logger'
   require 'oboe/util'
+  require 'base.rb'
   
   # If Oboe_metal is already defined then we are in a PaaS environment
   # with an alternate metal (such as Heroku: see the oboe-heroku gem)
   unless defined?(Oboe_metal)
-    if RUBY_PLATFORM == 'java'
-      require '/usr/local/tracelytics/tracelyticsagent.jar'
-      require 'joboe_metal'
-    else
-      require 'oboe_metal.so'
-      require 'oboe_metal'
+    begin
+      if RUBY_PLATFORM == 'java'
+        require 'joboe_metal'
+        require '/usr/local/tracelytics/tracelyticsagent.jar'
+      else
+        require 'oboe_metal'
+        require 'oboe_metal.so'
+      end
+    rescue LoadError
+      Oboe.loaded = false
+      $stderr.puts "=============================================================="
+      $stderr.puts "Missing TraceView libraries.  Tracing disabled."
+      $stderr.puts "See: https://support.tv.appneta.com/solution/articles/137973" 
+      $stderr.puts "=============================================================="
     end
   end
  
@@ -25,14 +34,10 @@ begin
   require 'oboe/ruby'
 
   # Frameworks
-  require 'oboe/frameworks/rails' if defined?(::Rails)
+  require 'oboe/frameworks/rails' if defined?(::Rails) and Oboe.loaded
 
-rescue LoadError
-  $stderr.puts "=============================================================="
-  $stderr.puts "Missing TraceView libraries.  Tracing disabled."
-  $stderr.puts "See: https://support.tv.appneta.com/solution/articles/137973" 
-  $stderr.puts "=============================================================="
 rescue Exception => e
   $stderr.puts "[oboe/error] Problem loading: #{e.inspect}"
   $stderr.puts e.backtrace
 end
+

data/lib/oboe/api/util.rb

@@ -1,6 +1,8 @@
 # Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
+require 'pp'
+
 module Oboe
   module API
     module Util
@@ -66,6 +68,50 @@ module Oboe
 
         false
       end
+      
+      # Internal: Pretty print a list of arguments for reporting
+      #
+      # args - the list of arguments to work on
+      #
+      # Returns a pretty string representation of arguments
+      def pps(*args)
+        old_out = $stdout
+        begin
+          s = StringIO.new
+          $stdout = s
+          pp(*args)
+        ensure
+          $stdout = old_out
+        end
+        s.string
+      end
+
+      # Internal: Determine a string to report representing klass
+      #
+      # args - an instance of a Class, a Class or a Module
+      #
+      # Returns a string representation of klass
+      def get_class_name(klass)
+        kv = {}
+        if klass.to_s =~ /::/
+          klass.class.to_s.rpartition('::').last
+        else
+          if klass.is_a?(Class) and klass.is_a?(Module)
+            # Class
+            kv["Class"] = klass.to_s
+        
+          elsif (not klass.is_a?(Class) and not klass.is_a?(Module))
+            # Class instance
+            kv["Class"] = klass.class.to_s
+          
+          else
+            # Module
+            kv["Module"] = klass.to_s
+          end
+        end
+        kv
+      end
+
     end
   end
 end

data/lib/oboe/config.rb

@@ -107,24 +107,13 @@ module Oboe
 
         # Assure value is an integer
         @@config[key.to_sym] = value.to_i
-
-        # Update liboboe with the new SampleRate value
-        Oboe::Context.setDefaultSampleRate(value.to_i)
+ 
+        Oboe.set_sample_rate(value)
       end
 
       # Update liboboe if updating :tracing_mode
       if key == :tracing_mode
-        case value.downcase
-        when 'never'
-          # OBOE_TRACE_NEVER
-          Oboe::Context.setTracingMode(0)
-        when 'always'
-          # OBOE_TRACE_ALWAYS
-          Oboe::Context.setTracingMode(1)
-        else
-          # OBOE_TRACE_THROUGH
-          Oboe::Context.setTracingMode(2)
-        end
+        Oboe.set_tracing_mode(value)
       end
     end
 

data/lib/oboe/inst/cassandra.rb

@@ -10,7 +10,7 @@ module Oboe
         begin
           report_kvs[:Op] = op.to_s
           report_kvs[:Cf] = column_family.to_s if column_family
-          report_kvs[:Key] = keys.to_s if keys
+          report_kvs[:Key] = keys.inspect if keys
          
           # Open issue - how to handle multiple Cassandra servers
           report_kvs[:RemoteHost], report_kvs[:RemotePort] = @servers.first.split(":")

data/lib/oboe/inst/mongo.rb

@@ -135,7 +135,7 @@ if defined?(::Mongo) and Oboe::Config[:mongo][:enabled]
             
             begin
               if m == :find_and_modify and args[0] and args[0].has_key?(:update)
-                report_kvs[:Update_Document] = args[0][:update]
+                report_kvs[:Update_Document] = args[0][:update].inspect
               end
 
               if m == :map_reduce

data/lib/oboe/inst/rack.rb

@@ -1,6 +1,8 @@
 # Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
+require 'uri'
+
 module Oboe
   class Rack
     attr_reader :app
@@ -14,28 +16,31 @@ module Oboe
 
       begin
         req = ::Rack::Request.new(env)
+
         report_kvs[:SampleRate]        = Oboe::Config[:sample_rate]
         report_kvs[:SampleSource]      = Oboe::Config[:sample_source]
         report_kvs['HTTP-Host']        = req.host
         report_kvs['Port']             = req.port
         report_kvs['Proto']            = req.scheme
-        report_kvs['Query-String']     = req.query_string unless req.query_string.empty?
-        report_kvs[:URL]               = req.path
+        report_kvs['Query-String']     = URI.unescape(req.query_string) unless req.query_string.empty?
+        report_kvs[:URL]               = URI.unescape(req.path)
         report_kvs[:Method]            = req.request_method
         report_kvs['AJAX']             = true if req.xhr?
         report_kvs['ClientIP']         = req.ip
 
-        report_kvs['X-TV-Meta']        = env['HTTP_X_TV_META']          if env.has_key?('HTTP_X_TV_META')
+        report_kvs['X-TV-Meta']         = env['HTTP_X_TV_META']          if env.has_key?('HTTP_X_TV_META')
 
         # Report any request queue'ing headers.  Report as 'Request-Start' or the summed Queue-Time
-        report_kvs['Request-Start']    = env['HTTP_X_REQUEST_START']    if env.has_key?('HTTP_X_REQUEST_START')
-        report_kvs['Request-Start']    = env['HTTP_X_QUEUE_START']      if env.has_key?('HTTP_X_QUEUE_START')
-        report_kvs['Queue-Time']       = env['HTTP_X_QUEUE_TIME']       if env.has_key?('HTTP_X_QUEUE_TIME')
-
-        report_kvs['Forwarded-For']    = env['HTTP_X_FORWARDED_FOR']    if env.has_key?('HTTP_X_FORWARDED_FOR')
-        report_kvs['Forwarded-Host']   = env['HTTP_X_FORWARDED_HOST']   if env.has_key?('HTTP_X_FORWARDED_HOST')
-        report_kvs['Forwarded-Proto']  = env['HTTP_X_FORWARDED_PROTO']  if env.has_key?('HTTP_X_FORWARDED_PROTO')
-        report_kvs['Forwarded-Port']   = env['HTTP_X_FORWARDED_PORT']   if env.has_key?('HTTP_X_FORWARDED_PORT')
+        report_kvs['Request-Start']     = env['HTTP_X_REQUEST_START']    if env.has_key?('HTTP_X_REQUEST_START')
+        report_kvs['Request-Start']     = env['HTTP_X_QUEUE_START']      if env.has_key?('HTTP_X_QUEUE_START')
+        report_kvs['Queue-Time']        = env['HTTP_X_QUEUE_TIME']       if env.has_key?('HTTP_X_QUEUE_TIME')
+
+        report_kvs['Forwarded-For']     = env['HTTP_X_FORWARDED_FOR']    if env.has_key?('HTTP_X_FORWARDED_FOR')
+        report_kvs['Forwarded-Host']    = env['HTTP_X_FORWARDED_HOST']   if env.has_key?('HTTP_X_FORWARDED_HOST')
+        report_kvs['Forwarded-Proto']   = env['HTTP_X_FORWARDED_PROTO']  if env.has_key?('HTTP_X_FORWARDED_PROTO')
+        report_kvs['Forwarded-Port']    = env['HTTP_X_FORWARDED_PORT']   if env.has_key?('HTTP_X_FORWARDED_PORT')
+
+        report_kvs['Ruby.Oboe.Version'] = ::Oboe::Version::STRING
       rescue Exception => e
         # Discard any potential exceptions. Debug log and report whatever we can.
         Oboe.logger.debug "[oboe/debug] Rack KV collection error: #{e.inspect}"

data/lib/oboe/inst/resque.rb

@@ -40,7 +40,7 @@ module Oboe
         if Oboe.tracing?
           report_kvs = extract_trace_details(:enqueue, klass, args)
 
-          Oboe::API.trace('resque', report_kvs, :enqueue) do
+          Oboe::API.trace('resque-client', report_kvs, :enqueue) do
             args.push({:parent_trace_id => Oboe::Context.toString}) if Oboe::Config[:resque][:link_workers]
             enqueue_without_oboe(klass, *args)
           end
@@ -54,7 +54,7 @@ module Oboe
           report_kvs = extract_trace_details(:enqueue_to, klass, args)
           report_kvs[:Queue] = queue.to_s if queue
 
-          Oboe::API.trace('resque', report_kvs) do
+          Oboe::API.trace('resque-client', report_kvs) do
             args.push({:parent_trace_id => Oboe::Context.toString}) if Oboe::Config[:resque][:link_workers]
             enqueue_to_without_oboe(queue, klass, *args)
           end
@@ -67,7 +67,7 @@ module Oboe
         if Oboe.tracing?
           report_kvs = extract_trace_details(:dequeue, klass, args)
 
-          Oboe::API.trace('resque', report_kvs) do
+          Oboe::API.trace('resque-client', report_kvs) do
             dequeue_without_oboe(klass, *args)
           end
         else
@@ -123,12 +123,12 @@ module Oboe
 
           # Force this trace regardless of sampling rate so that child trace can be
           # link to parent trace.
-          Oboe::API.start_trace('resque', nil, report_kvs.merge('Force' => true)) do
+          Oboe::API.start_trace('resque-worker', nil, report_kvs.merge('Force' => true)) do
             perform_without_oboe(job)
           end
 
         else
-          Oboe::API.start_trace('resque', nil, report_kvs) do
+          Oboe::API.start_trace('resque-worker', nil, report_kvs) do
             perform_without_oboe(job)
           end
         end

data/lib/oboe/version.rb

@@ -5,8 +5,8 @@ module Oboe
   module Version
     MAJOR = 2
     MINOR = 3
-    PATCH = 2
-    BUILD = nil
+    PATCH = 3
+    BUILD = 7
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
   end

data/lib/oboe_metal.rb

@@ -2,16 +2,10 @@
 # All rights reserved.
 
 module Oboe_metal
-  class Event
-    def self.metadataString(evt)
-      evt.metadataString()
-    end
-  end
-
   class Context
     class << self
       attr_accessor :layer_op
-
+      
       def log(layer, label, options = {}, with_backtrace = false)
         evt = Oboe::Context.createEvent()
         evt.addInfo("Layer", layer.to_s)
@@ -25,7 +19,7 @@ module Oboe_metal
 
         Oboe.reporter.sendReport(evt)
       end
-
+       
       def tracing_layer_op?(operation)
         if operation.is_a?(Array)
           return operation.include?(@layer_op)
@@ -35,22 +29,30 @@ module Oboe_metal
       end
     end
   end
-  
-  module Reporter
+
+  class Event
+    def self.metadataString(evt)
+      evt.metadataString()
+    end
+  end
+
+  class Reporter
     ##
     # Initialize the Oboe Context, reporter and report the initialization
     #
     def self.start
+      return unless Oboe.loaded
+
       begin
         Oboe_metal::Context.init() 
 
         if ENV['RACK_ENV'] == "test"
-          Oboe.reporter = Oboe::FileReporter.new("./tmp/trace_output.bson")
+          Oboe.reporter = Oboe::FileReporter.new("/tmp/trace_output.bson")
         else
           Oboe.reporter = Oboe::UdpReporter.new(Oboe::Config[:reporter_host])
         end
 
-        Oboe::API.report_init('ruby') unless ["development", "test"].include? ENV['RACK_ENV']
+        Oboe::API.report_init('rack') unless ["development", "test"].include? ENV['RACK_ENV']
       
       rescue Exception => e
         $stderr.puts e.message
@@ -64,28 +66,11 @@ module Oboe_metal
   end
 end
 
-module Oboe
+module Oboe 
+  extend OboeBase
   include Oboe_metal
 
   class << self
-    attr_accessor :reporter
- 
-    def always?
-      Oboe::Config[:tracing_mode].to_s == "always"
-    end
-    
-    def log(layer, label, options = {})
-      Context.log(layer, label, options = options)
-    end
-
-    def never?
-      Oboe::Config[:tracing_mode].to_s == "never"
-    end
-
-    def passthrough?
-      ["always", "through"].include?(Oboe::Config[:tracing_mode])
-    end
-      
     def sample?(opts = {})
       # Assure defaults since SWIG enforces Strings
       opts[:layer]      ||= ''
@@ -94,14 +79,36 @@ module Oboe
       Oboe::Context.sampleRequest(opts[:layer], opts[:xtrace], opts['X-TV-Meta'])
     end
 
-    def through?
-      Oboe::Config[:tracing_mode] == "through"
+    def set_tracing_mode(mode)
+      return unless Oboe.loaded
+
+      value = mode.to_sym
+
+      case value
+      when :never
+        # OBOE_TRACE_NEVER
+        Oboe::Context.setTracingMode(0)
+      when :always
+        # OBOE_TRACE_ALWAYS
+        Oboe::Context.setTracingMode(1)
+      when :through
+        # OBOE_TRACE_THROUGH
+        Oboe::Context.setTracingMode(2)
+      else
+        Oboe.logger.fatal "[oboe/error] Invalid tracing mode set: #{mode}"
+        # OBOE_TRACE_THROUGH
+        Oboe::Context.setTracingMode(2)
+      end
     end
-      
-    def tracing?
-      Oboe::Context.isValid and not Oboe.never?
+    
+    def set_sample_rate(rate)
+      if Oboe.loaded
+        # Update liboboe with the new SampleRate value
+        Oboe::Context.setDefaultSampleRate(rate.to_i)
+      end
     end
   end
 end
 
+Oboe.loaded = true
 

data/test/instrumentation/cassandra_test.rb

@@ -4,11 +4,29 @@ describe Oboe::Inst::Cassandra do
   before do
     clear_all_traces 
 
-    @client = Cassandra.new('TRCassInstr', '127.0.0.1:9160',
-                           :retries => 2, :connect_timeout => 1,
-                           :timeout => 5)
+    @client = Cassandra.new("system", "127.0.0.1:9160", { :timeout => 10 })
     @client.disable_node_auto_discovery!
 
+    @ks_name = "AppNetaCassandraTest"
+    
+    ks_def = CassandraThrift::KsDef.new(:name => @ks_name,
+              :strategy_class => "SimpleStrategy",
+              :strategy_options => { 'replication_factor' => '2' },
+              :cf_defs => [])
+      
+    @client.add_keyspace(ks_def) unless @client.keyspaces.include? @ks_name
+    @client.keyspace = @ks_name
+   
+    unless @client.column_families.include? "Users"
+      cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => "Users")
+      @client.add_column_family(cf_def)
+    end
+    
+    unless @client.column_families.include? "Statuses"
+      cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => "Statuses")
+      @client.add_column_family(cf_def)
+    end
+
     # These are standard entry/exit KVs that are passed up with all mongo operations
     @entry_kvs = {
       'Layer' => 'cassandra',
@@ -54,7 +72,7 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "insert"
     traces[1]['Cf'].must_equal "Users"
-    traces[1]['Key'].must_equal "5"
+    traces[1]['Key'].must_equal "\"5\""
     traces[1]['Consistency'].must_equal "1"
     traces[1]['Ttl'].must_equal "600"
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
@@ -74,7 +92,7 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "remove"
     traces[1]['Cf'].must_equal "Users"
-    traces[1]['Key'].must_equal "5"
+    traces[1]['Key'].must_equal "\"5\""
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
   end
@@ -94,7 +112,7 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "count_columns"
     traces[1]['Cf'].must_equal "Statuses"
-    traces[1]['Key'].must_equal "12"
+    traces[1]['Key'].must_equal "\"12\""
     traces[1]['Count'].must_equal "50"
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
@@ -113,7 +131,7 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "get_columns"
     traces[1]['Cf'].must_equal "Statuses"
-    traces[1]['Key'].must_equal "12"
+    traces[1]['Key'].must_equal "\"12\""
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
   end
@@ -149,7 +167,7 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "get"
     traces[1]['Cf'].must_equal "Statuses"
-    traces[1]['Key'].must_equal "12"
+    traces[1]['Key'].must_equal "\"12\""
     traces[1]['Reversed'].must_equal "true"
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
@@ -169,13 +187,13 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "exists?"
     traces[1]['Cf'].must_equal "Statuses"
-    traces[1]['Key'].must_equal "12"
+    traces[1]['Key'].must_equal "\"12\""
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
     
     traces[3]['Op'].must_equal "exists?"
     traces[3]['Cf'].must_equal "Statuses"
-    traces[3]['Key'].must_equal "12"
+    traces[3]['Key'].must_equal "\"12\""
     traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
   end
 
@@ -196,37 +214,53 @@ describe Oboe::Inst::Cassandra do
     validate_event_keys(traces[2], @exit_kvs)
   end
 
-  it 'should trace create_index and drop_index' do
+  it 'should trace create_index' do
     Oboe::API.start_trace('cassandra_test', '', {}) do
-      @client.create_index('TRCassInstr', 'Statuses', 'x', 'LongType')
-      @client.drop_index('TRCassInstr', 'Statuses', 'x')
+      @client.create_index(@ks_name, 'Statuses', 'column_name', 'LongType')
     end
-    
+
     traces = get_all_traces
     
-    traces.count.must_equal 6
+    traces.count.must_equal 4
     validate_outer_layers(traces, 'cassandra_test')
 
     validate_event_keys(traces[1], @entry_kvs)
     traces[1]['Op'].must_equal "create_index"
     traces[1]['Cf'].must_equal "Statuses"
-    traces[1]['Keyspace'].must_equal "TRCassInstr"
-    traces[1]['Column_name'].must_equal "x"
+    traces[1]['Keyspace'].must_equal @ks_name
+    traces[1]['Column_name'].must_equal "column_name"
     traces[1]['Validation_class'].must_equal "LongType"
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
     
-    validate_event_keys(traces[3], @entry_kvs)
-    traces[3]['Op'].must_equal "drop_index"
-    traces[3]['Cf'].must_equal "Statuses"
-    traces[3]['Keyspace'].must_equal "TRCassInstr"
-    traces[3]['Column_name'].must_equal "x"
-    traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
-    validate_event_keys(traces[4], @exit_kvs)
+    # Clean up
+    @client.drop_index(@ks_name, 'Statuses', 'column_name')
+  end
+  
+  it 'should trace drop_index' do
+    # Prep
+    @client.create_index(@ks_name, 'Statuses', 'column_name', 'LongType')
+
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.drop_index(@ks_name, 'Statuses', 'column_name')
+    end
+    
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "drop_index"
+    traces[1]['Cf'].must_equal "Statuses"
+    traces[1]['Keyspace'].must_equal @ks_name
+    traces[1]['Column_name'].must_equal "column_name"
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
   end
 
   it 'should trace get_indexed_slices' do
-    @client.create_index('TRCassInstr', 'Statuses', 'x', 'LongType')
+    @client.create_index(@ks_name, 'Statuses', 'x', 'LongType')
     Oboe::API.start_trace('cassandra_test', '', {}) do
       expressions   =  [
                          { :column_name => 'x',
@@ -252,7 +286,7 @@ describe Oboe::Inst::Cassandra do
   
   it 'should trace add and remove of column family' do
     cf_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
-    cf_def = CassandraThrift::CfDef.new(:keyspace => "TRCassInstr", :name => cf_name)
+    cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => cf_name)
 
     Oboe::API.start_trace('cassandra_test', '', {}) do
       @client.add_column_family(cf_def)
@@ -274,23 +308,22 @@ describe Oboe::Inst::Cassandra do
     traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
   end
   
-  it 'should trace add and remove of keyspace' do
+  it 'should trace adding a keyspace' do
     ks_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
     column_families = [{:name =>"a"}, {:name => "b", :type => :super}]
     ks_def = CassandraThrift::KsDef.new(:name => ks_name,
               :strategy_class => "org.apache.cassandra.locator.SimpleStrategy",
-              :replication_factor => 1,
+              :strategy_options => { 'replication_factor' => '2' },
               :cf_defs => [])
 
     Oboe::API.start_trace('cassandra_test', '', {}) do
       @client.add_keyspace(ks_def)
       @client.keyspace = ks_name
-      @client.drop_keyspace(ks_name)
     end
     
     traces = get_all_traces
     
-    traces.count.must_equal 6
+    traces.count.must_equal 4
     validate_outer_layers(traces, 'cassandra_test')
 
     validate_event_keys(traces[1], @entry_kvs)
@@ -298,10 +331,23 @@ describe Oboe::Inst::Cassandra do
     traces[1]['Name'].must_equal ks_name
     traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
     validate_event_keys(traces[2], @exit_kvs)
+  end
+  
+  it 'should trace the removal of a keyspace' do
+    Oboe::API.start_trace('cassandra_test', '', {}) do
+      @client.drop_keyspace(@ks_name)
+    end
     
-    traces[3]['Op'].must_equal "drop_keyspace"
-    traces[3]['Name'].must_equal ks_name
-    traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    traces = get_all_traces
+    
+    traces.count.must_equal 4
+    validate_outer_layers(traces, 'cassandra_test')
+
+    validate_event_keys(traces[1], @entry_kvs)
+    traces[1]['Op'].must_equal "drop_keyspace"
+    traces[1]['Name'].must_equal @ks_name
+    traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    validate_event_keys(traces[2], @exit_kvs)
   end
   
   it "should obey :collect_backtraces setting when true" do