cassandra-utils 0.2.1 → 0.3.1.pre.beta.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12d130d6d4d0301a5b1c7d20f5e74118f46c37f5
-  data.tar.gz: 2c1f1aa3aaa0c753deeb77c27a91895e8bddc6cf
+  metadata.gz: 4e551a8b992a71ae2b24834faef52d77f61137eb
+  data.tar.gz: 002209ac50837ebd0e4a542b86bcdb442b2073be
 SHA512:
-  metadata.gz: a5aa7e037428fc7410d28ae231f74a7e6aa2241b5f6777243383338f74478afbe7df3c6a1ecad435f869359cf8bdefbf458d4e0229170593d77cdfb2ce146ba2
-  data.tar.gz: ebfe0dcd4fe290c800a79108a1452a0ac2beeaaa1996d6562e7454b24c0a0c6e541e8cba2d00d47c7dda0a8a6d892fc183e2e0ef629805f2ecefa4c8e10bdc4d
+  metadata.gz: b2f744c06e90c15931aba8bd6a9bfdabd4dfef6f5469e83ccc681928f6bcbde06067e0ada434c774b8df9482af046c795d74708eb337186e89296822cf98af67
+  data.tar.gz: 982bb9611cc120b08493f160b07dcda482cda4022c361126d1999c63ab066babd4f94cd409388ea2219b19cf6788f5386ed8feec2d0e421086db96f9a0114e3c

data/.gitignore

@@ -6,3 +6,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.swp

data/Gemfile

@@ -3,6 +3,9 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in cassandra-utils.gemspec
 gemspec
 
+# TODO: Remove this when the next version of diplomat is released
+gem 'diplomat', github: 'WeAreFarmGeek/diplomat', ref: 'e64b0dec3b3616ded40bb64139a388dfc1a68232'
+
 group :development do
   gem 'bundler', '~> 1.7'
   gem 'thor-scmversion', '= 1.7.0'

data/Gemfile.lock

@@ -1,3 +1,12 @@
+GIT
+  remote: git://github.com/WeAreFarmGeek/diplomat.git
+  revision: e64b0dec3b3616ded40bb64139a388dfc1a68232
+  ref: e64b0dec3b3616ded40bb64139a388dfc1a68232
+  specs:
+    diplomat (1.1.0)
+      faraday (~> 0.9)
+      json
+
 PATH
   remote: .
   specs:
@@ -10,10 +19,16 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    daemon_runner (0.2.2)
+    daemon_runner (0.4.1)
+      diplomat (~> 1.0)
       logging (~> 2.1)
       mixlib-shellout (~> 2.2)
+      retryable (~> 2.0)
+      rufus-scheduler (~> 3.2)
     dogstatsd-ruby (1.6.0)
+    faraday (0.10.0)
+      multipart-post (>= 1.2, < 3)
+    json (2.0.2)
     little-plugger (1.1.4)
     logging (2.1.0)
       little-plugger (~> 1.1)
@@ -21,7 +36,10 @@ GEM
     minitest (5.9.0)
     mixlib-shellout (2.2.7)
     multi_json (1.12.1)
+    multipart-post (2.0.0)
     rake (10.5.0)
+    retryable (2.0.4)
+    rufus-scheduler (3.2.2)
     thor (0.19.1)
     thor-scmversion (1.7.0)
       mixlib-shellout
@@ -33,6 +51,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.7)
   cassandra-utils!
+  diplomat!
   minitest (~> 5.0)
   rake (~> 10.0)
   thor-scmversion (= 1.7.0)

data/bin/cass-util

@@ -1,17 +1,29 @@
 #!/usr/bin/env ruby
 require_relative '../lib/cassandra/utils'
+require_relative '../lib/cassandra/tasks'
 require 'thor'
 
 class CassandraUtils < Thor
 
-  method_option :loop_sleep_time, type: :numeric,
+  class_option :loop_sleep_time, type: :numeric,
     required: true, default: 120,
     desc: 'Frequency tasks are run'
-  desc 'stats', 'Write metrics to Datadog'
-  def stats
+  class_option :cleanup_service_name, type: :string,
+    required: true, default: 'cassandra',
+    desc: 'Unique string to be used in obtaining a Semaphore.  Example: cassandra-#{cluster_name}'
+  class_option :cleanup_lock_count, type: :numeric,
+    required: true, default: 1,
+    desc: 'Number of nodes that can obtain a Semaphore lock'
+  desc 'util', 'Perform various utilities'
+  def util
     s = ::Cassandra::Utils::Daemon.new(options)
     s.start!
   end
+  # Backwards compatibility
+  desc 'stats', '[DEPRECATED - Use util] Write metrics to Datadog'
+  def stats
+    send(:util)
+  end
 end
 
 CassandraUtils.start

data/lib/cassandra/tasks/autoclean.rb

@@ -0,0 +1,257 @@
+require 'socket'
+require 'json'
+require 'time'
+require 'set'
+require 'tmpdir'
+require_relative '../utils/version'
+
+module Cassandra
+  module Tasks
+   class Autoclean
+     include ::DaemonRunner::Logger
+
+     # @return [String] the path on disk where tokens will be cached
+     attr_reader :token_cache_path
+
+     # Create a new Autoclean task
+     #
+     # @param options [Object] optional configuration settings
+     # (see #token_cache_path)
+     #
+     # @return [Autoclean]
+     #
+     def initialize(options = {})
+       @token_cache_path = options[:token_cache_path]
+       @token_cache_path ||= File.join(Dir.tmpdir, 'autoclean-tokens.json')
+       @service_name = options[:cleanup_service_name]
+       @lock_count = options[:cleanup_lock_count]
+     end
+
+     # Schedule the Cassandra cleanup process to run daily
+     #
+     def schedule
+       [:interval, '1d']
+     end
+
+     # Return the status of the Cassandra node
+     #
+     # A node is considered up if it has a status of "Up" as reported by
+     # "nodetool status". If multiple nodes with this node's IP address show
+     # up in "nodetool status", this node is considered down.
+     #
+     # @return [:up, :down]
+     #
+     def status
+       return(:down).tap { logger.warn 'Cassandra node is DOWN' } if address.nil?
+       results = (nodetool_status || '').split("\n")
+       results.map! { |line| line.strip }
+       results.select! { |line| line.include? address }
+       results.map! { |line| line.split(/\s+/)[0] }
+       results.compact!
+       return(:down).tap do
+         logger.warn "Cannot find the Cassandra node (#{address}) in `nodetool status`"
+       end if results.size != 1
+       (results.first[0] == 'U') ? :up : :down
+     end
+
+     # Return the state of the Cassandra node
+     #
+     # The returned state is reported by "nodetool netstats".
+     #
+     # @return [state, nil]
+     #
+     def state
+       results = (nodetool_netstats || '').split("\n")
+       results.map! { |line| line.strip }
+       results.select! { |line| line.include? 'Mode:' }
+       results.map! { |line| line.split(':')[1] }
+       results.compact!
+       return nil if results.size != 1
+       results.first.strip.downcase.to_sym
+     end
+
+     # Run the Cassandra cleanup process if necessary
+     #
+     def run!
+       return unless status == :up
+       return unless state == :normal
+
+       new_tokens = Set.new tokens
+       old_tokens = Set.new cached_tokens
+       return if new_tokens == old_tokens
+
+       ::DaemonRunner::Semaphore.lock(@service_name, @lock_count) do
+         result = nodetool_cleanup
+         save_tokens if !result.nil? && result.exitstatus == 0
+       end
+     end
+
+     # Get the cached tokens this node owns
+     #
+     # @return [Array<String>] Cached tokens
+     #
+     def cached_tokens
+       data = token_cache.read
+       data = JSON.parse data
+       return [] unless data['version'] == ::Cassandra::Utils::VERSION
+
+       tokens = data['tokens']
+       return [] if tokens.nil?
+       return [] unless tokens.respond_to? :each
+
+       tokens.sort!
+       tokens
+     # Token file could not be opend or parsed
+     rescue Errno::ENOENT, JSON::ParserError
+       []
+     end
+
+     # Save the list of tokens this node owns to disk
+     # These can be read by `cached_tokens`
+     #
+     def save_tokens
+       data = {
+         :timestamp => Time.now.iso8601,
+         :tokens => tokens,
+         :version => ::Cassandra::Utils::VERSION
+       }
+
+       token_cache.write data.to_json
+       token_cache.flush
+     end
+
+     # Get the tokens this node owns
+     #
+     # The "nodetool ring" command returns
+     #
+     # Address    Rack  Status  State   Load  Size  Owns  Token
+     # 127.0.0.1  r1    Up      Normal  10    GB    33%   123456789
+     #
+     # @return [Array<String>] Tokens owned by this node
+     #
+     def tokens
+       return [] if address.nil?
+       results = (nodetool_ring || '').split("\n")
+       results.map! { |line| line.strip }
+       results.select! { |line| line.start_with? address }
+       results.map! { |line| line.split(/\s+/)[7] }
+       results.compact!
+       results.sort
+     end
+
+     # Get the IP address of this node
+     #
+     # @return [String, nil] IP address of this node
+     #
+     def address
+       if @address.nil?
+         addr = Socket.ip_address_list.find { |addr| addr.ipv4_private? }
+         @address = addr.ip_address unless addr.nil?
+       end
+       @address
+     end
+
+     def task_id
+       ['autoclean', 'nodetool']
+     end
+
+     private
+
+     # Run the "nodetool ring" command and return the output
+     #
+     # @return [String, nil] Output from the "nodetool ring" command
+     #
+     def nodetool_ring
+       @nodetool_ring ||= DaemonRunner::ShellOut.new(command: 'nodetool ring', timeout: 300)
+       @nodetool_ring.run!
+       @nodetool_ring.stdout
+     end
+
+     # Run the "nodetool status' command and return the output
+     #
+     # @return [String, nil] Output from the "nodetool status" command
+     #
+     def nodetool_status
+       @nodetool_status ||= DaemonRunner::ShellOut.new(command: 'nodetool status', timeout: 300)
+       @nodetool_status.run!
+       @nodetool_status.stdout
+     end
+
+     # Run the "nodetool netstats' command and return the output
+     #
+     # @return [String, nil] Output from the "nodetool netstats" command
+     #
+     def nodetool_netstats
+       @nodetool_netstats ||= DaemonRunner::ShellOut.new(command: 'nodetool netstats', timeout: 300)
+       @nodetool_netstats.run!
+       @nodetool_netstats.stdout
+     end
+
+     # Get the status of a "nodetool cleanup" command
+     #
+     # This will atempt to track a running "nodetool cleanup" process if one's
+     # found. If a running process isn't found, a new process will be launched.
+     #
+     # @return [Process::Status, nil]
+     #
+     def nodetool_cleanup
+       pid = find_nodetool_cleanup
+       if pid
+         logger.debug "Found nodetool cleanup process #{pid} already running"
+         Utils::Statsd.new('cassandra.cleanup.running').push!(1)
+       end
+       pid = exec_nodetool_cleanup
+       if pid
+         logger.debug "Started nodetool cleanup process #{pid}"
+         Utils::Statsd.new('cassandra.cleanup.running').push!(1)
+         status = wait_nodetool_cleanup pid
+         logger.debug "Completed nodetool cleanup process #{pid}"
+       end
+       status
+     end
+
+     # Get the ID of the first running "nodetool cleanup" process found
+     #
+     # @return [Integer, nil]
+     #
+     def find_nodetool_cleanup
+       @pgrep_nodetool_cleanup ||= ::DaemonRunner::ShellOut.new(command: 'pgrep -f "NodeCmd.+cleanu[p]"', valid_exit_codes: [0,1])
+       @pgrep_nodetool_cleanup.run!
+       pids = @pgrep_nodetool_cleanup.stdout.strip.split "\n"
+       return nil if pids.empty?
+       pids.first.to_i
+     end
+
+     # Run "nodetool cleanup" command
+     #
+     # @return [Integer] ID of the "nodetool cleanup" command
+     #
+     def exec_nodetool_cleanup
+       # The `pgroup: true` option spawns cleanup in its own process group.
+       # So if this process dies, cleanup continues to run.
+       @nodetool_cleanup ||= ::DaemonRunner::ShellOut.new(command: 'nodetool cleanup', wait: false)
+       @nodetool_cleanup.run!
+     end
+
+     # Wait for a "nodetool cleanup" process to exit
+     #
+     # This handles the `SystemCallError` that's raised if no child process is
+     # found. In that case, the returned status will be `nil`.
+     #
+     # @return [Process::Status, nil] status
+     #
+     def wait_nodetool_cleanup pid
+       logger.debug "Waiting for nodetool cleanup process #{pid} to complete"
+       ::DaemonRunner::ShellOut.wait2(pid, Process::WUNTRACED)
+     end
+
+     # Get the cache tokens wil be saved in
+     #
+     # @return [File] File where tokens wil be saved
+     #
+     def token_cache
+       File.new(token_cache_path, 'w+')
+     end
+   end
+  end
+end

data/lib/cassandra/tasks.rb

@@ -0,0 +1 @@
+require_relative 'tasks/autoclean'

data/lib/cassandra/utils/cli/base.rb

@@ -1,5 +1,4 @@
 require 'mixlib/shellout'
-require 'statsd'
 
 module Cassandra
   module Utils
@@ -12,7 +11,7 @@ module Cassandra
         end
 
         def timeout
-          15
+          300
         end
 
         def runner
@@ -29,23 +28,9 @@ module Cassandra
           @command.error!
           @stdout = @command.stdout
           out = output
-          push_metric(out)
+          Utils::Statsd.new(metric_name).to_dd(out).push!
           out
         end
-
-        protected
-
-        def statsd
-          @statsd ||= ::Statsd.new('localhost', 8125)
-        end
-
-        def push_metric(value)
-          statsd.gauge(metric_name, value)
-        end
-
-        def to_dd(out)
-          out == true ? 1 : 0
-        end
       end
     end
   end

data/lib/cassandra/utils/daemon.rb

@@ -6,10 +6,30 @@ module Cassandra
 
       def tasks
         [
-          [::Cassandra::Utils::Stats::Compaction.new, 'run!'],
-          [::Cassandra::Utils::Stats::Cleanup.new, 'run!']
+          [auto_clean_task, 'run!'],
+          [health_stat, 'run!'],
+          [compaction_stat, 'run!'],
+          [cleanup_stat, 'run!']
         ]
       end
+
+      private
+
+      def auto_clean_task
+        @auto_clean_task ||= ::Cassandra::Tasks::Autoclean.new(options)
+      end
+
+      def health_stat
+        @health_stat ||= ::Cassandra::Utils::Stats::Health.new
+      end
+
+      def compaction_stat
+        @compaction_stat ||= ::Cassandra::Utils::Stats::Compaction.new
+      end
+
+      def cleanup_stat
+        @cleanup_stat ||= ::Cassandra::Utils::Stats::Cleanup.new
+      end
     end
   end
 end

data/lib/cassandra/utils/stats/cleanup.rb

@@ -9,12 +9,15 @@ module Cassandra
 
         def output
           cleanup = stdout.lines.any? { |l| l.include?('Cleanup') }
-          to_dd(cleanup)
         end
 
         def metric_name
           'cassandra.cleanup.running'
         end
+
+        def task_id
+          ['cleanup', 'nodetool']
+        end
       end
     end
   end

data/lib/cassandra/utils/stats/compaction.rb

@@ -9,12 +9,15 @@ module Cassandra
 
         def output
           compaction = stdout.lines.any? { |l| l.include?('Compaction') }
-          to_dd(compaction)
         end
 
         def metric_name
           'cassandra.compaction.running'
         end
+
+        def task_id
+          ['compaction', 'nodetool']
+        end
       end
     end
   end

data/lib/cassandra/utils/stats/health.rb

@@ -0,0 +1,73 @@
+module Cassandra
+  module Utils
+    module Stats
+      class Health < Utils::CLI::Base
+        def run!
+          running = true
+          if state == :normal
+            running &&= nodetool_statusgossip.strip == 'running'
+            running &&= nodetool_statusthrift.strip == 'running'
+          end
+          Utils::Statsd.new(metric_name).to_dd(running).push!
+          running
+        end
+
+        def metric_name
+          'cassandra.service.running'
+        end
+
+        # Return the state of the Cassandra node
+        #
+        # The returned state is reported by "nodetool netstats".
+        #
+        # @return [state, nil]
+        #
+        def state
+          results = (nodetool_netstats || '').split("\n")
+          results.map! { |line| line.strip }
+          results.select! { |line| line.include? 'Mode:' }
+          results.map! { |line| line.split(':')[1] }
+          results.compact!
+          return nil if results.size != 1
+          results.first.strip.downcase.to_sym
+        end
+
+        def task_id
+          ['health', 'nodetool']
+        end
+
+        private
+
+        # Run the "nodetool statusgossip' command and return the output
+        #
+        # @return [String, nil] Output from the "nodetool statusgossip" command
+        #
+        def nodetool_statusgossip
+          @nodetool_statusgossip ||= DaemonRunner::ShellOut.new(command: 'nodetool statusgossip')
+          @nodetool_statusgossip.run!
+          @nodetool_statusgossip.stdout
+        end
+
+        # Run the "nodetool statusthrift' command and return the output
+        #
+        # @return [String, nil] Output from the "nodetool statusthrift" command
+        #
+        def nodetool_statusthrift
+          @nodetool_statusthrift||= DaemonRunner::ShellOut.new(command: 'nodetool statusthrift')
+          @nodetool_statusthrift.run!
+          @nodetool_statusthrift.stdout
+        end
+
+        # Run the "nodetool netstats' command and return the output
+        #
+        # @return [String, nil] Output from the "nodetool netstats" command
+        #
+        def nodetool_netstats
+          @nodetool_netstats ||= DaemonRunner::ShellOut.new(command: 'nodetool netstats', timeout: 300)
+          @nodetool_netstats.run!
+          @nodetool_netstats.stdout
+        end
+      end
+    end
+  end
+end

data/lib/cassandra/utils/stats.rb

@@ -1,2 +1,3 @@
 require_relative 'stats/compaction'
 require_relative 'stats/cleanup'
+require_relative 'stats/health'

data/lib/cassandra/utils/statsd.rb

@@ -0,0 +1,24 @@
+require 'statsd'
+
+module Cassandra
+  module Utils
+    class Statsd
+      attr_reader :statsd, :metric_name, :value
+
+      def initialize(metric_name)
+        @statsd ||= ::Statsd.new('localhost', 8125)
+        @metric_name = metric_name
+        self
+      end
+
+      def to_dd(value)
+        @value = (value == true ? 1 : 0)
+        self
+      end
+
+      def push!(value = @value)
+        statsd.gauge(metric_name, value)
+      end
+    end
+  end
+end

data/lib/cassandra/utils.rb

@@ -1,3 +1,4 @@
 require_relative 'utils/cli/base'
 require_relative 'utils/daemon'
+require_relative 'utils/statsd'
 require_relative 'utils/stats'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cassandra-utils
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1.pre.beta.1
 platform: ruby
 authors:
 - Andrew Thompson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-12 00:00:00.000000000 Z
+date: 2016-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mixlib-shellout
@@ -129,12 +129,16 @@ files:
 - bin/console
 - bin/setup
 - cassandra-utils.gemspec
+- lib/cassandra/tasks.rb
+- lib/cassandra/tasks/autoclean.rb
 - lib/cassandra/utils.rb
 - lib/cassandra/utils/cli/base.rb
 - lib/cassandra/utils/daemon.rb
 - lib/cassandra/utils/stats.rb
 - lib/cassandra/utils/stats/cleanup.rb
 - lib/cassandra/utils/stats/compaction.rb
+- lib/cassandra/utils/stats/health.rb
+- lib/cassandra/utils/statsd.rb
 - lib/cassandra/utils/version.rb
 homepage: https://github.com/rapid7/cassandra-utils
 licenses:
@@ -151,9 +155,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.3