oboe 2.7.0.3-java

data/lib/rails/generators/oboe/install_generator.rb

@@ -0,0 +1,76 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
+module Oboe
+  class InstallGenerator < ::Rails::Generators::Base
+    source_root File.join(File.dirname(__FILE__), 'templates')
+    desc "Copies an oboe initializer files to your application."
+
+    def copy_initializer
+      # Set defaults
+      @tracing_mode = 'through'
+      @sample_rate = '300000'
+      @verbose = 'false'
+
+      print_header
+
+      while true do
+        print_body
+
+        user_tracing_mode = ask shell.set_color "* Tracing Mode? [through]:", :yellow
+        user_tracing_mode.downcase!
+
+        break if user_tracing_mode.blank?
+        valid = ['always', 'through', 'never'].include?(user_tracing_mode)
+
+        say shell.set_color "Valid values are 'always', 'through' or 'never'", :red, :bold unless valid
+        if valid
+          @tracing_mode = user_tracing_mode
+          break
+        end
+      end
+
+      print_footer
+
+      template "oboe_initializer.rb", "config/initializers/oboe.rb"
+    end
+
+    private
+
+      def print_header
+        say ""
+        say shell.set_color "Welcome to the TraceView Ruby instrumentation setup.", :green, :bold
+        say ""
+        say "To instrument your Rails application, you can setup your tracing strategy here."
+        say ""
+        say shell.set_color "Documentation Links", :magenta
+        say "-------------------"
+        say ""
+        say "Details on configuring your sample rate:"
+        say "https://support.appneta.com/cloud/configuring-sampling"
+        say ""
+        say "More information on instrumenting Ruby applications can be found here:"
+        say "https://support.appneta.com/cloud/installing-ruby-instrumentation"
+      end
+
+      def print_body
+        say ""
+        say shell.set_color "Tracing Mode", :magenta
+        say "------------"
+        say "Tracing Mode determines when traces should be initiated for incoming requests.  Valid"
+        say "options are #{shell.set_color "always", :yellow}, #{shell.set_color "through", :yellow} (when using an instrumented Apache or Nginx) and #{shell.set_color "never", :yellow}."
+        say ""
+        say "If you're not using an instrumented Apache or Nginx, set this directive to #{shell.set_color "always", :yellow} in"
+        say "order to initiate tracing from Ruby."
+        say ""
+      end
+
+      def print_footer
+        say ""
+        say "You can change configuration values in the future by modifying config/initializers/oboe.rb"
+        say ""
+        say "Thanks! Creating the TraceView initializer..."
+        say ""
+      end
+  end
+end

data/lib/rails/generators/oboe/templates/oboe_initializer.rb

@@ -0,0 +1,94 @@
+# AppNeta TraceView Initializer (the oboe gem)
+# http://www.appneta.com/products/traceview/
+#
+# Details on configuring your sample rate:
+# https://support.appneta.com/cloud/configuring-sampling
+#
+# More information on instrumenting Ruby applications can be found here:
+# https://support.appneta.com/cloud/installing-ruby-instrumentation
+
+if defined?(Oboe::Config)
+  # Tracing Mode determines when traces should be initiated for incoming requests.  Valid
+  # options are always, through (when using an instrumented Apache or Nginx) and never.
+  #
+  # If you're not using an instrumented Apache or Nginx, set this directive to always in
+  # order to initiate tracing from Ruby.
+  Oboe::Config[:tracing_mode] = '<%= @tracing_mode %>'
+
+  # sample_rate is a value from 0 - 1m indicating the fraction of requests per million to trace
+  # Oboe::Config[:sample_rate] = <%= @sample_rate %>
+
+  # Verbose output of instrumentation initialization
+  # Oboe::Config[:verbose] = <%= @verbose %>
+
+  #
+  # Resque Options
+  #
+  # :link_workers - associates Resque enqueue operations with the jobs they queue by piggybacking
+  #                 an additional argument on the Redis queue that is stripped prior to job
+  #                 processing
+  #                 !!! Note: Make sure both the enqueue side and the Resque workers are instrumented
+  #                 before enabling this or jobs will fail !!!
+  #                 (Default: false)
+  # Oboe::Config[:resque][:link_workers] = false
+  #
+  # Set to true to disable Resque argument logging (Default: false)
+  # Oboe::Config[:resque][:log_args] = false
+
+  # The oboe Ruby client has the ability to sanitize query literals
+  # from SQL statements.  By default this is disabled.  Enable to
+  # avoid collecting and reporting query literals to TraceView.
+  # Oboe::Config[:sanitize_sql] = false
+
+  #
+  # Enabling/Disabling Instrumentation
+  #
+  # If you're having trouble with one of the instrumentation libraries, they
+  # can be individually disabled here by setting the :enabled
+  # value to false:
+  #
+  # Oboe::Config[:action_controller][:enabled] = true
+  # Oboe::Config[:active_record][:enabled] = true
+  # Oboe::Config[:action_view][:enabled] = true
+  # Oboe::Config[:cassandra][:enabled] = true
+  # Oboe::Config[:dalli][:enabled] = true
+  # Oboe::Config[:memcache][:enabled] = true
+  # Oboe::Config[:memcached][:enabled] = true
+  # Oboe::Config[:mongo][:enabled] = true
+  # Oboe::Config[:moped][:enabled] = true
+  # Oboe::Config[:nethttp][:enabled] = true
+  # Oboe::Config[:redis][:enabled] = true
+  # Oboe::Config[:resque][:enabled] = true
+
+  #
+  # Enabling/Disabling Backtrace Collection
+  #
+  # Instrumentation can optionally collect backtraces as they collect
+  # performance metrics.  Note that this has a negative impact on
+  # performance but can be useful when trying to locate the source of
+  # a certain call or operation.
+  #
+  # Oboe::Config[:action_controller][:collect_backtraces] = true
+  # Oboe::Config[:active_record][:collect_backtraces] = true
+  # Oboe::Config[:action_view][:collect_backtraces] = true
+  # Oboe::Config[:cassandra][:collect_backtraces] = true
+  # Oboe::Config[:dalli][:collect_backtraces] = false
+  # Oboe::Config[:memcache][:collect_backtraces] = false
+  # Oboe::Config[:memcached][:collect_backtraces] = false
+  # Oboe::Config[:mongo][:collect_backtraces] = true
+  # Oboe::Config[:moped][:collect_backtraces] = true
+  # Oboe::Config[:nethttp][:collect_backtraces] = true
+  # Oboe::Config[:redis][:collect_backtraces] = false
+  # Oboe::Config[:resque][:collect_backtraces] = true
+  #
+
+  #
+  # Blacklist actions
+  #
+  # e.g. if your load balancer requests 'index#ok'
+  #
+  #   Oboe::Config[:action_blacklist] = {
+  #     'index#ok' => true
+  # }
+
+end

data/oboe.gemspec

@@ -0,0 +1,29 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "oboe/version"
+
+Gem::Specification.new do |s|
+  s.name = %q{oboe}
+  s.version = Oboe::Version::STRING
+  s.date = Time.now.strftime('%Y-%m-%d')
+
+  s.license = "AppNeta Open License, Version 1.0"
+
+  s.authors = ["Peter Giacomo Lombardo", "Spiros Eliopoulos"]
+  s.email = %q{traceviewsupport@appneta.com}
+  s.homepage = %q{http://www.appneta.com/products/traceview/}
+  s.summary = %q{AppNeta TraceView performance instrumentation gem for Ruby}
+  s.description = %q{The oboe gem provides TraceView instrumentation for MRI Ruby, JRuby and related frameworks.}
+
+  s.extra_rdoc_files = ["LICENSE"]
+  s.files = `git ls-files`.split("\n")
+  s.test_files  = Dir.glob("{test}/**/*.rb")
+
+  s.platform   = defined?(JRUBY_VERSION) ? 'java' : Gem::Platform::RUBY
+  s.extensions = ['ext/oboe_metal/extconf.rb'] unless defined?(JRUBY_VERSION)
+
+  s.add_runtime_dependency('json', '>= 0')
+  s.add_development_dependency('rake', '>= 0')
+
+  s.required_ruby_version = '>= 1.8.6'
+end
+

data/release.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# for use only when you're ready to push from prod -> rubygems
+#
+
+if [ $# -ne 0 ]
+then
+  echo -e "Usage: `basename $0`"
+  echo -e "-Summary-"
+  echo -e "\t This script will help you build and release a new version of the oboe Ruby gem."
+  echo -e "\t It will also create a git tag with the version being released."
+  echo -e ""
+  echo -e "-Steps-"
+  echo -e "\t 1. Update lib/oboe/version.rb with the new version you wish to release."
+  echo -e "\t 2. Re-run this script without any arguments."
+  echo -e ""
+  echo -e "-Notes-"
+  echo -e "\t You must have ~/.gem/credentials setup with the Appneta API key for Rubygems."
+  echo -e "\t The API key should be titled :rubygems_appneta: in the credentials file."
+  echo -e ""
+  echo -e "\t Gems with letters in the build number (e.g. pre1 or beta1)will be released "
+  echo -e "\t as a prerelease gem on Rubygems."
+  echo -e ""
+  echo -e "\t See here for an explanation on prelease gems:"
+  echo -e "\t http://guides.rubygems.org/patterns/#prerelease-gems"
+  exit $E_BADARGS
+fi
+
+if [ $(git branch -a | grep ^* | awk '{print $2}') != "prod" ]; then
+  echo -e "You can only release gems from prod branch."
+  echo -e "Do a 'git checkout prod' and try again."
+  exit
+fi
+
+#set -e # stop on first non-zero exit code
+#set -x # show commands as they happen
+
+# Get gem version from lib/oboe/version.rb
+VERSION=`/usr/bin/env ruby ./get_version.rb`
+
+read -p "Are you sure you want to release oboe gem version $VERSION to Rubygems? [y/N]" -n 1 -r
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+  echo -e ""
+  
+  # tag release (if tag already exists, bails out)
+  echo -e "Creating git tag rel-$VERSION..."
+  if ! git tag rel-$VERSION; then
+    echo -e "Couldn't create tag for ${VERSION}: if it already exists, you need to bump the version."
+    exit
+  fi
+
+  echo -e "Pushing tags to origin (Github)..."
+  git push --tags
+
+  # Build and publish the gem to Rubygems.org
+  echo -e "Building gem..."
+  gem build oboe.gemspec
+  echo -e "Pushing built gem to Rubygems..."
+  gem push -k rubygems_appneta oboe-$VERSION.gem
+else
+  echo -e ""
+  echo -e "Canceled...nothing done.  Have a nice day."
+fi
+

data/test/frameworks/apps/grape_simple.rb

@@ -0,0 +1,10 @@
+require 'grape'
+
+class GrapeSimple < Grape::API
+  set :reload, true
+
+  get '/json_endpoint' do
+    present({ :test => true })
+  end
+end
+

data/test/frameworks/apps/padrino_simple.rb

@@ -0,0 +1,41 @@
+# This test Padrino stack file was taken from the padrino-core gem.
+#
+PADRINO_ROOT = File.dirname(__FILE__) unless defined? PADRINO_ROOT
+# Remove this comment if you want do some like this: ruby PADRINO_ENV=test app.rb
+#
+# require 'rubygems'
+# require 'padrino-core'
+#
+
+class SimpleDemo < Padrino::Application
+  set :public_folder, File.dirname(__FILE__)
+  set :reload, true
+  before { true }
+  after  { true }
+  error(404) { "404" }
+end
+
+SimpleDemo.controllers do
+  get "/" do
+    'The magick number is: 2767356926488785838763860464013972991031534522105386787489885890443740254365!' # Change only the number!!!
+  end
+
+  get "/rand" do
+    rand(2 ** 256).to_s
+  end
+
+  get "/render" do
+    render :erb, "This is an erb render"
+  end
+end
+
+## If you want use this as a standalone app uncomment:
+#
+# Padrino.mount("SimpleDemo").to("/")
+# Padrino.run! unless Padrino.loaded? # If you enable reloader prevent to re-run the app
+#
+# Then run it from your console: ruby -I"lib" test/fixtures/apps/simple.rb
+#
+
+Padrino.load!
+

data/test/frameworks/apps/sinatra_simple.rb

@@ -0,0 +1,20 @@
+require 'sinatra'
+
+class SinatraSimple < Sinatra::Base
+  set :reload, true
+
+  get "/" do
+    'The magick number is: 2767356926488785838763860464013972991031534522105386787489885890443740254365!' # Change only the number!!!
+  end
+
+  get "/rand" do
+    rand(2 ** 256).to_s
+  end
+
+  get "/render" do
+    render :erb, "This is an erb render"
+  end
+end
+
+use SinatraSimple
+

data/test/frameworks/grape_test.rb

@@ -0,0 +1,27 @@
+if RUBY_VERSION >= '1.9.3'
+  require 'minitest_helper'
+  require File.expand_path(File.dirname(__FILE__) + '/apps/grape_simple')
+
+  describe Grape do
+    before do
+      clear_all_traces
+    end
+
+    it "should trace a request to a simple grape stack" do
+      @app = GrapeSimple
+
+      r = get "/json_endpoint"
+
+      traces = get_all_traces
+      traces.count.must_equal 5
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[1]['Layer'].must_equal "grape"
+      traces[2]['Layer'].must_equal "grape"
+      traces[2].has_key?('Controller').must_equal true
+      traces[2].has_key?('Action').must_equal true
+      traces[3]['Label'].must_equal "info"
+    end
+  end
+end

data/test/frameworks/padrino_test.rb

@@ -0,0 +1,25 @@
+if RUBY_VERSION >= '1.9.3'
+  require "minitest_helper"
+  require File.expand_path(File.dirname(__FILE__) + '/apps/padrino_simple')
+
+  describe Padrino do
+    before do
+      clear_all_traces
+    end
+
+    it "should trace a request to a simple padrino stack" do
+      @app = SimpleDemo
+
+      r = get "/render"
+
+      traces = get_all_traces
+      traces.count.must_equal 9
+
+      validate_outer_layers(traces, 'rack')
+
+      traces[1]['Layer'].must_equal "padrino"
+      traces[6]['Controller'].must_equal "SimpleDemo"
+      traces[7]['Label'].must_equal "info"
+    end
+  end
+end

data/test/frameworks/sinatra_test.rb

@@ -0,0 +1,25 @@
+require "minitest_helper"
+require File.expand_path(File.dirname(__FILE__) + '/apps/sinatra_simple')
+
+describe Sinatra do
+  before do
+    clear_all_traces
+  end
+
+  it "should trace a request to a simple sinatra stack" do
+    @app = SinatraSimple
+
+    r = get "/render"
+
+    traces = get_all_traces
+    traces.count.must_equal 9
+
+    validate_outer_layers(traces, 'rack')
+
+    traces[1]['Layer'].must_equal "sinatra"
+    traces[3]['Label'].must_equal "profile_entry"
+    traces[6]['Controller'].must_equal "SinatraSimple"
+    traces[7]['Label'].must_equal "info"
+  end
+end
+

data/test/instrumentation/cassandra_test.rb

@@ -0,0 +1,381 @@
+require 'minitest_helper'
+
+# The cassandra-rb client doesn't support JRuby
+# https://github.com/cassandra-rb/cassandra
+unless defined?(JRUBY_VERSION)
+  describe Oboe::Inst::Cassandra do
+    before do
+      clear_all_traces
+
+      @client = Cassandra.new("system", "127.0.0.1:9160", { :timeout => 10 })
+      @client.disable_node_auto_discovery!
+
+      @ks_name = "AppNetaCassandraTest"
+
+      ks_def = CassandraThrift::KsDef.new(:name => @ks_name,
+                :strategy_class => "SimpleStrategy",
+                :strategy_options => { 'replication_factor' => '2' },
+                :cf_defs => [])
+
+      @client.add_keyspace(ks_def) unless @client.keyspaces.include? @ks_name
+      @client.keyspace = @ks_name
+
+      unless @client.column_families.include? "Users"
+        cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => "Users")
+        @client.add_column_family(cf_def)
+      end
+
+      unless @client.column_families.include? "Statuses"
+        cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => "Statuses")
+        @client.add_column_family(cf_def)
+      end
+
+      # These are standard entry/exit KVs that are passed up with all mongo operations
+      @entry_kvs = {
+        'Layer' => 'cassandra',
+        'Label' => 'entry',
+        'RemoteHost' => '127.0.0.1',
+        'RemotePort' => '9160' }
+
+      @exit_kvs = { 'Layer' => 'cassandra', 'Label' => 'exit' }
+      @collect_backtraces = Oboe::Config[:cassandra][:collect_backtraces]
+    end
+
+    after do
+      Oboe::Config[:cassandra][:collect_backtraces] = @collect_backtraces
+      @client.disconnect!
+    end
+
+    it 'Stock Cassandra should be loaded, defined and ready' do
+      defined?(::Cassandra).wont_match nil
+    end
+
+    it 'Cassandra should have oboe methods defined' do
+      [ :insert, :remove, :count_columns, :get_columns, :multi_get_columns, :get,
+        :multi_get, :get_range_single, :get_range_batch, :get_indexed_slices,
+        :create_index, :drop_index, :add_column_family, :drop_column_family,
+        :add_keyspace, :drop_keyspace ].each do |m|
+        ::Cassandra.method_defined?("#{m}_with_oboe").must_equal true
+      end
+      # Special 'exists?' case
+      ::Cassandra.method_defined?("exists_with_oboe?").must_equal true
+    end
+
+    it 'should trace insert' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        user = {'screen_name' => 'larry', "blah" => "ok"}
+        @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "insert"
+      traces[1]['Cf'].must_equal "Users"
+      traces[1]['Key'].must_equal "\"5\""
+      traces[1]['Consistency'].must_equal "1"
+      traces[1]['Ttl'].must_equal "600"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace remove' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.remove(:Users, '5', 'blah')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "remove"
+      traces[1]['Cf'].must_equal "Users"
+      traces[1]['Key'].must_equal "\"5\""
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace count_columns' do
+      @client.insert(:Statuses, '12', {'body' => 'v1', 'user' => 'v2'})
+
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.count_columns(:Statuses, '12', :count => 50)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "count_columns"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1]['Count'].must_equal "50"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace get_columns' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.get_columns(:Statuses, '12', ['body'])
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get_columns"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace multi_get_columns' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.multi_get_columns(:Users, ['12', '5'], ['body'])
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "multi_get_columns"
+      traces[1]['Cf'].must_equal "Users"
+      traces[1]['Key'].must_equal "[\"12\", \"5\"]"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace get' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.get(:Statuses, '12', :reversed => true)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1]['Reversed'].must_equal "true"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace exists?' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.exists?(:Statuses, '12')
+        @client.exists?(:Statuses, '12', 'body')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 6
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "exists?"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Key'].must_equal "\"12\""
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+
+      traces[3]['Op'].must_equal "exists?"
+      traces[3]['Cf'].must_equal "Statuses"
+      traces[3]['Key'].must_equal "\"12\""
+      traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    end
+
+    it 'should trace get_range_keys' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.get_range_keys(:Statuses, :key_count => 4)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get_range_batch"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace create_index' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.create_index(@ks_name, 'Statuses', 'column_name', 'LongType')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "create_index"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Keyspace'].must_equal @ks_name
+      traces[1]['Column_name'].must_equal "column_name"
+      traces[1]['Validation_class'].must_equal "LongType"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+
+      # Clean up
+      @client.drop_index(@ks_name, 'Statuses', 'column_name')
+    end
+
+    it 'should trace drop_index' do
+      # Prep
+      @client.create_index(@ks_name, 'Statuses', 'column_name', 'LongType')
+
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.drop_index(@ks_name, 'Statuses', 'column_name')
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "drop_index"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1]['Keyspace'].must_equal @ks_name
+      traces[1]['Column_name'].must_equal "column_name"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace get_indexed_slices' do
+      @client.create_index(@ks_name, 'Statuses', 'x', 'LongType')
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        expressions   =  [
+                           { :column_name => 'x',
+                             :value => [0,20].pack("NN"),
+                             :comparison => "=="},
+                           { :column_name => 'non_indexed',
+                             :value => [5].pack("N*"),
+                             :comparison => ">"} ]
+        @client.get_indexed_slices(:Statuses, expressions).length
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "get_indexed_slices"
+      traces[1]['Cf'].must_equal "Statuses"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace add and remove of column family' do
+      cf_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
+      cf_def = CassandraThrift::CfDef.new(:keyspace => @ks_name, :name => cf_name)
+
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.add_column_family(cf_def)
+        @client.drop_column_family(cf_name)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 6
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "add_column_family"
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+
+      traces[3]['Op'].must_equal "drop_column_family"
+      traces[3]['Cf'].must_equal cf_name
+      traces[3].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+    end
+
+    it 'should trace adding a keyspace' do
+      ks_name = (0...10).map{ ('a'..'z').to_a[rand(26)] }.join
+      column_families = [{:name =>"a"}, {:name => "b", :type => :super}]
+      ks_def = CassandraThrift::KsDef.new(:name => ks_name,
+                :strategy_class => "org.apache.cassandra.locator.SimpleStrategy",
+                :strategy_options => { 'replication_factor' => '2' },
+                :cf_defs => [])
+
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.add_keyspace(ks_def)
+        @client.keyspace = ks_name
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "add_keyspace"
+      traces[1]['Name'].must_equal ks_name
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it 'should trace the removal of a keyspace' do
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        @client.drop_keyspace(@ks_name)
+      end
+
+      traces = get_all_traces
+
+      traces.count.must_equal 4
+      validate_outer_layers(traces, 'cassandra_test')
+
+      validate_event_keys(traces[1], @entry_kvs)
+      traces[1]['Op'].must_equal "drop_keyspace"
+      traces[1]['Name'].must_equal @ks_name
+      traces[1].has_key?('Backtrace').must_equal Oboe::Config[:cassandra][:collect_backtraces]
+      validate_event_keys(traces[2], @exit_kvs)
+    end
+
+    it "should obey :collect_backtraces setting when true" do
+      Oboe::Config[:cassandra][:collect_backtraces] = true
+
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        user = {'screen_name' => 'larry', "blah" => "ok"}
+        @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+      end
+
+      traces = get_all_traces
+      layer_has_key(traces, 'cassandra', 'Backtrace')
+    end
+
+    it "should obey :collect_backtraces setting when false" do
+      Oboe::Config[:cassandra][:collect_backtraces] = false
+
+      Oboe::API.start_trace('cassandra_test', '', {}) do
+        user = {'screen_name' => 'larry', "blah" => "ok"}
+        @client.insert(:Users, '5', user, { :ttl => 600, :consistency => 1})
+      end
+
+      traces = get_all_traces
+      layer_doesnt_have_key(traces, 'cassandra', 'Backtrace')
+    end
+
+  end
+end