traceview 3.0.0

data/lib/traceview/version.rb

@@ -0,0 +1,16 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  ##
+  # The current version of the gem.  Used mainly by
+  # traceview.gemspec during gem build process
+  module Version
+    MAJOR = 3
+    MINOR = 0
+    PATCH = 0
+    BUILD = nil
+
+    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
+  end
+end

data/lib/traceview/xtrace.rb

@@ -0,0 +1,90 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  ##
+  # Methods to act on, manipulate or investigate an X-Trace
+  # value
+  module XTrace
+    class << self
+      ##
+      #  TraceView::XTrace.valid?
+      #
+      #  Perform basic validation on a potential X-Trace ID
+      #
+      def valid?(xtrace)
+        # Shouldn't be nil
+        return false unless xtrace
+
+        # The X-Trace ID shouldn't be an initialized empty ID
+        return false if (xtrace =~ /^1b0000000/i) == 0
+
+        # Valid X-Trace IDs have a length of 58 bytes and start with '1b'
+        return false unless xtrace.length == 58 && (xtrace =~ /^1b/i) == 0
+
+        true
+      rescue StandardError => e
+        TraceView.logger.debug e.message
+        TraceView.logger.debug e.backtrace
+        false
+      end
+
+      ##
+      # TraceView::XTrace.task_id
+      #
+      # Extract and return the task_id portion of an X-Trace ID
+      #
+      def task_id(xtrace)
+        return nil unless TraceView::XTrace.valid?(xtrace)
+
+        xtrace[2..41]
+      rescue StandardError => e
+        TraceView.logger.debug e.message
+        TraceView.logger.debug e.backtrace
+        return nil
+      end
+
+      ##
+      # TraceView::XTrace.edge_id
+      #
+      # Extract and return the edge_id portion of an X-Trace ID
+      #
+      def edge_id(xtrace)
+        return nil unless TraceView::XTrace.valid?(xtrace)
+
+        xtrace[42..57]
+      rescue StandardError => e
+        TraceView.logger.debug e.message
+        TraceView.logger.debug e.backtrace
+        return nil
+      end
+
+      ##
+      # continue_service_context
+      #
+      # In the case of service calls such as external HTTP requests, we
+      # pass along X-Trace headers so that request context can be maintained
+      # across servers and applications.
+      #
+      # Remote requests can return a X-Trace header in which case we want
+      # to pickup on and continue the context in most cases.
+      #
+      # @start is the context just before the outgoing request
+      #
+      # @finish is the context returned to us (as an HTTP response header
+      # if that be the case)
+      #
+      def continue_service_context(start, finish)
+        if TraceView::XTrace.valid?(finish) && TraceView.tracing?
+
+          # Assure that we received back a valid X-Trace with the same task_id
+          if TraceView::XTrace.task_id(start) == TraceView::XTrace.task_id(finish)
+            TraceView::Context.fromString(finish)
+          else
+            TraceView.logger.debug "Mismatched returned X-Trace ID: #{finish}"
+          end
+        end
+      end
+    end
+  end
+end

data/test/frameworks/apps/grape_nested.rb

@@ -0,0 +1,30 @@
+require 'grape'
+
+class Wrapper < Grape::API
+  class GrapeSimple < Grape::API
+    rescue_from :all do |e|
+      error_response({ message: "rescued from #{e.class.name}" })
+    end
+
+    get '/json_endpoint' do
+      present({ :test => true })
+    end
+
+    get "/break" do
+      raise Exception.new("This should have http status code 500!")
+    end
+
+    get "/error" do
+      error!("This is a error with 'error'!")
+    end
+
+    get "/breakstring" do
+      raise "This should have http status code 500!"
+    end
+  end
+end
+
+class GrapeNested < Grape::API
+  mount Wrapper::GrapeSimple
+end
+

data/test/frameworks/apps/grape_simple.rb

@@ -0,0 +1,24 @@
+require 'grape'
+
+class GrapeSimple < Grape::API
+  rescue_from :all do |e|
+    error_response({ message: "rescued from #{e.class.name}" })
+  end
+
+  get '/json_endpoint' do
+    present({ :test => true })
+  end
+
+  get "/break" do
+    raise Exception.new("This should have http status code 500!")
+  end
+
+  get "/error" do
+    error!("This is a error with 'error'!")
+  end
+
+  get "/breakstring" do
+    raise "This should have http status code 500!"
+  end
+end
+

data/test/frameworks/apps/padrino_simple.rb

@@ -0,0 +1,45 @@
+# This test Padrino stack file was taken from the padrino-core gem.
+#
+PADRINO_ROOT = File.dirname(__FILE__) unless defined? PADRINO_ROOT
+# Remove this comment if you want do some like this: ruby PADRINO_ENV=test app.rb
+#
+# require 'rubygems'
+# require 'padrino-core'
+#
+
+class SimpleDemo < Padrino::Application
+  set :public_folder, File.dirname(__FILE__)
+  set :reload, true
+  before { true }
+  after  { true }
+  error(404) { "404" }
+end
+
+SimpleDemo.controllers do
+  get "/" do
+    'The magick number is: 2767356926488785838763860464013972991031534522105386787489885890443740254365!' # Change only the number!!!
+  end
+
+  get "/rand" do
+    rand(2 ** 256).to_s
+  end
+
+  get "/render" do
+    render :erb, "This is an erb render"
+  end
+
+  get "/break" do
+    raise "This is a controller exception!"
+  end
+end
+
+## If you want use this as a standalone app uncomment:
+#
+# Padrino.mount("SimpleDemo").to("/")
+# Padrino.run! unless Padrino.loaded? # If you enable reloader prevent to re-run the app
+#
+# Then run it from your console: ruby -I"lib" test/fixtures/apps/simple.rb
+#
+
+Padrino.load!
+

data/test/frameworks/apps/sinatra_simple.rb

@@ -0,0 +1,24 @@
+require 'sinatra'
+
+class SinatraSimple < Sinatra::Base
+  set :reload, true
+
+  get "/" do
+    'The magick number is: 2767356926488785838763860464013972991031534522105386787489885890443740254365!' # Change only the number!!!
+  end
+
+  get "/rand" do
+    rand(2 ** 256).to_s
+  end
+
+  get "/render" do
+    render :erb, "This is an erb render"
+  end
+
+  get "/break" do
+    raise "This is a controller exception!"
+  end
+end
+
+use SinatraSimple
+

data/test/frameworks/grape_test.rb

@@ -0,0 +1,142 @@
+if RUBY_VERSION >= '1.9.3'
+  require 'minitest_helper'
+  require File.expand_path(File.dirname(__FILE__) + '/apps/grape_simple')
+  require File.expand_path(File.dirname(__FILE__) + '/apps/grape_nested')
+
+  describe Grape do
+    before do
+      clear_all_traces
+    end
+
+    it "should trace a request to a simple grape stack" do
+      @app = GrapeSimple
+
+      r = get "/json_endpoint"
+
+      r.status.must_equal 200
+      r.headers.key?('X-Trace').must_equal true
+
+      traces = get_all_traces
+      traces.count.must_equal 5
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[2]['Layer'].must_equal "grape"
+      traces[3]['Layer'].must_equal "grape"
+      traces[3].has_key?('Controller').must_equal true
+      traces[3].has_key?('Action').must_equal true
+      traces[4]['Label'].must_equal "exit"
+
+      # Validate the existence of the response header
+      r.headers.key?('X-Trace').must_equal true
+      r.headers['X-Trace'].must_equal traces[4]['X-Trace']
+    end
+
+    it "should trace a request to a nested grape stack" do
+      @app = GrapeNested
+
+      r = get "/json_endpoint"
+
+      r.status.must_equal 200
+      r.headers.key?('X-Trace').must_equal true
+
+      traces = get_all_traces
+      traces.count.must_equal 5
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[2]['Layer'].must_equal "grape"
+      traces[3]['Layer'].must_equal "grape"
+      traces[3].has_key?('Controller').must_equal true
+      traces[3].has_key?('Action').must_equal true
+      traces[4]['Label'].must_equal "exit"
+
+      # Validate the existence of the response header
+      r.headers.key?('X-Trace').must_equal true
+      r.headers['X-Trace'].must_equal traces[4]['X-Trace']
+    end
+
+    it "should trace a an error in a nested grape stack" do
+      @app = GrapeNested
+
+      r = get "/error"
+
+      r.status.must_equal 500
+      r.headers.key?('X-Trace').must_equal true
+
+      traces = get_all_traces
+      traces.count.must_equal 6
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[2]['Layer'].must_equal "grape"
+      traces[2]['Label'].must_equal "entry"
+      traces[3]['Layer'].must_equal "grape"
+      traces[3]['Label'].must_equal "exit"
+      traces[3].has_key?('Controller').must_equal true
+      traces[3].has_key?('Action').must_equal true
+      traces[4]['Label'].must_equal "error"
+      traces[4]['ErrorClass'].must_equal "GrapeError"
+      traces[4]['ErrorMsg'].must_equal "This is a error with 'error'!"
+      traces[4].has_key?('Backtrace').must_equal true
+      traces[5]['Layer'].must_equal "rack"
+      traces[5]['Label'].must_equal "exit"
+
+      # Validate the existence of the response header
+      r.headers.key?('X-Trace').must_equal true
+      r.headers['X-Trace'].must_equal traces[5]['X-Trace']
+    end
+
+
+    it "should trace a request with an exception" do
+      @app = GrapeSimple
+
+      begin
+        r = get "/break"
+      rescue Exception => e
+        # Do not handle/raise this error so
+        # we can continue to test
+      end
+
+      traces = get_all_traces
+      traces.count.must_equal 6
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[2]['Layer'].must_equal "grape"
+      traces[3]['Layer'].must_equal "grape"
+      traces[3].has_key?('Controller').must_equal true
+      traces[3].has_key?('Action').must_equal true
+      traces[4]['Label'].must_equal "error"
+      traces[4]['ErrorClass'].must_equal "Exception"
+      traces[4]['ErrorMsg'].must_equal "This should have http status code 500!"
+      traces[5]['Label'].must_equal "exit"
+    end
+
+    it "should trace a request with an error" do
+      @app = GrapeSimple
+
+      r = get "/error"
+
+      traces = get_all_traces
+      traces.count.must_equal 6
+
+      r.status.must_equal 500
+      r.headers.key?('X-Trace').must_equal true
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[0]['Layer'].must_equal "rack"
+      traces[2]['Layer'].must_equal "grape"
+      traces[3]['Layer'].must_equal "grape"
+      traces[3].has_key?('Controller').must_equal true
+      traces[3].has_key?('Action').must_equal true
+      traces[4]['Label'].must_equal "error"
+      traces[4]['ErrorClass'].must_equal "GrapeError"
+      traces[4]['ErrorMsg'].must_equal "This is a error with 'error'!"
+      traces[5]['Layer'].must_equal "rack"
+      traces[5]['Label'].must_equal "exit"
+      traces[5]['Status'].must_equal 500
+    end
+  end
+end

data/test/frameworks/padrino_test.rb

@@ -0,0 +1,30 @@
+if RUBY_VERSION >= '1.9.3'
+  require "minitest_helper"
+  require File.expand_path(File.dirname(__FILE__) + '/apps/padrino_simple')
+
+  describe Padrino do
+    before do
+      clear_all_traces
+    end
+
+    it "should trace a request to a simple padrino stack" do
+      @app = SimpleDemo
+
+      r = get "/render"
+
+      traces = get_all_traces
+
+      traces.count.must_equal 9
+      valid_edges?(traces).must_equal true
+      validate_outer_layers(traces, 'rack')
+
+      traces[2]['Layer'].must_equal "padrino"
+      traces[7]['Controller'].must_equal "SimpleDemo"
+      traces[8]['Label'].must_equal "exit"
+
+      # Validate the existence of the response header
+      r.headers.key?('X-Trace').must_equal true
+      r.headers['X-Trace'].must_equal traces[8]['X-Trace']
+    end
+  end
+end

data/test/frameworks/sinatra_test.rb

@@ -0,0 +1,30 @@
+require "minitest_helper"
+require File.expand_path(File.dirname(__FILE__) + '/apps/sinatra_simple')
+
+describe Sinatra do
+  before do
+    clear_all_traces
+  end
+
+  it "should trace a request to a simple sinatra stack" do
+    @app = SinatraSimple
+
+    r = get "/render"
+
+    traces = get_all_traces
+
+    traces.count.must_equal 9
+    valid_edges?(traces).must_equal true
+    validate_outer_layers(traces, 'rack')
+
+    traces[2]['Layer'].must_equal "sinatra"
+    traces[4]['Label'].must_equal "profile_entry"
+    traces[7]['Controller'].must_equal "SinatraSimple"
+    traces[8]['Label'].must_equal "exit"
+
+    # Validate the existence of the response header
+    r.headers.key?('X-Trace').must_equal true
+    r.headers['X-Trace'].must_equal traces[8]['X-Trace']
+  end
+end
+

data/test/instrumentation/cassandra_test.rb

@@ -0,0 +1,380 @@
+require 'minitest_helper'
+
+# The cassandra-rb client doesn't support JRuby
+# https://github.com/cassandra-rb/cassandra
+unless defined?(JRUBY_VERSION)
+  describe "Cassandra" do
+    before do
+      clear_all_traces
+
+      @client = Cassandra.new("system", "127.0.0.1:9160", { :timeout => 10 })
+      @client.disable_node_auto_discovery!
+
+      @ks_name = "AppNetaCassandraTest"
+
+      ks_def = CassandraThrift::KsDef.new(:name => @ks_name,
+                :strategy_class => "SimpleStrategy",
+                :strategy_options => { 'replication_factor' => '2' },
+                :cf_defs => [])
+
+      @client.add_keyspace(ks_def) unless @client.keyspaces.include? @ks_name
+      @client.keyspace = @ks_name
+
+      unless @client.column_families.include? "Users"
+        cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => "Users")
+        @client.add_column_family(cf_def)
+      end
+
+      unless @client.column_families.include? "Statuses"
+        cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => "Statuses")
+        @client.add_column_family(cf_def)
+      end
+
+      # These are standard entry/exit KVs that are passed up with all mongo operations
+      @entry_kvs = {
+        'Layer' => 'cassandra',
+        'Label' => 'entry',
+        'RemoteHost' => '127.0.0.1',
+        'RemotePort' => '9160' }
+
+      @exit_kvs = { 'Layer' => 'cassandra', 'Label' => 'exit' }
+      @collect_backtraces = TraceView::Config[:cassandra][:collect_backtraces]
+    end
+
+    after do
+      TraceView::Config[:cassandra][:collect_backtraces] = @collect_backtraces
+      @client.disconnect!
+    end
+
+    it 'Stock Cassandra should be loaded, defined and ready' do
+      defined?(::Cassandra).wont_match nil
+    end
+
+    it 'Cassandra should have traceview methods defined' do
+      [ :insert, :remove, :count_columns, :get_columns, :multi_get_columns, :get,
+        :multi_get, :get_range_single, :get_range_batch, :get_indexed_slices,
+        :create_index, :drop_index, :add_column_family, :drop_column_family,
+        :add_keyspace, :drop_keyspace ].each do |m|
+        ::Cassandra.method_defined?("#{m}_with_traceview").must_equal true
+      end
+      # Special 'exists?' case
+      ::Cassandra.method_defined?("exists_with_traceview?").must_equal true
+    end
+
+    it 'should trace insert' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        user = {'screen_name' => 'larry', "blah" => "ok"}
+        @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "insert"
+      traces[1]['Cf'].must_equal "Users"
+      traces[1]['Key'].must_equal "\"5\""
+      traces[1]['Consistency'].must_equal 1
+      traces[1]['Ttl'].must_equal 600
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace remove' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.remove(:Users, '5', 'blah')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "remove"
+      traces[1]['Cf'].must_equal "Users"
+      traces[1]['Key'].must_equal "\"5\""
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace count_columns' do
+      @client.insert(:Statuses, '12', {'body' => 'v1', 'user' => 'v2'})
+
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.count_columns(:Statuses, '12', :count => 50)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "count_columns"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1]['Count'].must_equal 50
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace get_columns' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.get_columns(:Statuses, '12', ['body'])
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get_columns"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace multi_get_columns' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.multi_get_columns(:Users, ['12', '5'], ['body'])
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "multi_get_columns"
+      traces[1]['Cf'].must_equal "Users"
+      traces[1]['Key'].must_equal "[\"12\", \"5\"]"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace get' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.get(:Statuses, '12', :reversed => true)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1]['Reversed'].must_equal "true"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace exists?' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.exists?(:Statuses, '12')
+        @client.exists?(:Statuses, '12', 'body')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 6
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "exists?"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+
+      traces[3]['Op'].must_equal "exists?"
+      traces[3]['Cf'].must_equal "Statuses"
+      traces[3]['Key'].must_equal "\"12\""
+      traces[3].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+    end
+
+    it 'should trace get_range_keys' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.get_range_keys(:Statuses, :key_count => 4)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get_range_batch"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace create_index' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.create_index(@ks_name, 'Statuses', 'column_name', 'LongType')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "create_index"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Keyspace'].must_equal @ks_name
+      traces[1]['Column_name'].must_equal "column_name"
+      traces[1]['Validation_class'].must_equal "LongType"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+
+      # Clean up
+      @client.drop_index(@ks_name, 'Statuses', 'column_name')
+    end
+
+    it 'should trace drop_index' do
+      # Prep
+      @client.create_index(@ks_name, 'Statuses', 'column_name', 'LongType')
+
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.drop_index(@ks_name, 'Statuses', 'column_name')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "drop_index"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Keyspace'].must_equal @ks_name
+      traces[1]['Column_name'].must_equal "column_name"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace get_indexed_slices' do
+      @client.create_index(@ks_name, 'Statuses', 'x', 'LongType')
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        expressions   =  [
+                           { :column_name => 'x',
+                             :value => [0,20].pack("NN"),
+                             :comparison => "=="},
+                           { :column_name => 'non_indexed',
+                             :value => [5].pack("N*"),
+                             :comparison => ">"} ]
+        @client.get_indexed_slices(:Statuses, expressions).length
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get_indexed_slices"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace add and remove of column family' do
+      cf_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
+      cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => cf_name)
+
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.add_column_family(cf_def)
+        @client.drop_column_family(cf_name)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 6
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "add_column_family"
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+
+      traces[3]['Op'].must_equal "drop_column_family"
+      traces[3]['Cf'].must_equal cf_name
+      traces[3].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+    end
+
+    it 'should trace adding a keyspace' do
+      ks_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
+      ks_def = CassandraThrift::KsDef.new(:name => ks_name,
+                :strategy_class => "org.apache.cassandra.locator.SimpleStrategy",
+                :strategy_options => { 'replication_factor' => '2' },
+                :cf_defs => [])
+
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.add_keyspace(ks_def)
+        @client.keyspace = ks_name
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "add_keyspace"
+      traces[1]['Name'].must_equal ks_name
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace the removal of a keyspace' do
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        @client.drop_keyspace(@ks_name)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "drop_keyspace"
+      traces[1]['Name'].must_equal @ks_name
+      traces[1].has_key?('Backtrace').must_equal TraceView::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it "should obey :collect_backtraces setting when true" do
+      TraceView::Config[:cassandra][:collect_backtraces] = true
+
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        user = {'screen_name' => 'larry', "blah" => "ok"}
+        @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+      end
+
+      traces = get_all_traces
+      layer_has_key(traces, 'cassandra', 'Backtrace')
+    end
+
+    it "should obey :collect_backtraces setting when false" do
+      TraceView::Config[:cassandra][:collect_backtraces] = false
+
+      TraceView::API.start_trace('cassandra_test', '', {}) do
+        user = {'screen_name' => 'larry', "blah" => "ok"}
+        @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+      end
+
+      traces = get_all_traces
+      layer_doesnt_have_key(traces, 'cassandra', 'Backtrace')
+    end
+
+  end
+end