graphiterb 0.1.5 → 0.2.0

data/Rakefile

@@ -10,6 +10,7 @@ begin
     gem.email = "info@infochimps.org"
     gem.homepage = "http://github.com/infochimps/graphiterb"
     gem.authors = ["Philip (flip) Kromer (@mrflip)"]
+    gem.add_dependency "configliere", ">= 0"
     gem.add_development_dependency "rspec", ">= 1.2.9"
     gem.add_development_dependency "yard", ">= 0"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.2.0

data/graphiterb.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{graphiterb}
-  s.version = "0.1.5"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer (@mrflip)"]
-  s.date = %q{2010-08-16}
+  s.date = %q{2010-08-17}
   s.description = %q{Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com}
   s.email = %q{info@infochimps.org}
   s.extra_rdoc_files = [
@@ -32,10 +32,15 @@ Gem::Specification.new do |s|
      "examples/toy.rb",
      "graphiterb.gemspec",
      "lib/graphiterb.rb",
-     "lib/graphiterb/graphite_logger.rb",
-     "lib/graphiterb/graphite_script.rb",
-     "lib/graphiterb/graphite_sender.rb",
-     "lib/graphiterb/graphite_system_logger.rb",
+     "lib/graphiterb/monitors.rb",
+     "lib/graphiterb/monitors/directory_tree.rb",
+     "lib/graphiterb/monitors/disk_space.rb",
+     "lib/graphiterb/script.rb",
+     "lib/graphiterb/sender.rb",
+     "lib/graphiterb/utils.rb",
+     "lib/graphiterb/utils/extensions.rb",
+     "lib/graphiterb/utils/log.rb",
+     "lib/graphiterb/utils/system.rb",
      "spec/graphiterb_spec.rb",
      "spec/spec.opts",
      "spec/spec_helper.rb"
@@ -48,11 +53,11 @@ Gem::Specification.new do |s|
   s.test_files = [
     "spec/graphiterb_spec.rb",
      "spec/spec_helper.rb",
+     "examples/toy.rb",
      "examples/storage_monitor.rb",
-     "examples/api_call_monitor.rb",
      "examples/loadavg_graphite_sender.rb",
-     "examples/toy.rb",
-     "examples/file_monitor.rb"
+     "examples/file_monitor.rb",
+     "examples/api_call_monitor.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -60,13 +65,16 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<configliere>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_development_dependency(%q<yard>, [">= 0"])
     else
+      s.add_dependency(%q<configliere>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<yard>, [">= 0"])
     end
   else
+    s.add_dependency(%q<configliere>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
     s.add_dependency(%q<yard>, [">= 0"])
   end

data/lib/graphiterb/monitors/directory_tree.rb

@@ -0,0 +1,123 @@
+module Graphiterb
+  module Monitors
+
+    # A class for monitoring the contents of a directory.
+    #
+    # Will only monitor files modified within the last hour by
+    # default.
+    class DirectoryTree < Graphiterb::Monitors::PeriodicMonitor
+
+      # The root of the directory tree being monitored.
+      attr_accessor :root
+
+      # A regular expression filter that must be matched by files
+      # within the root directory to be counted.
+      attr_accessor :filter_re
+
+      def initialize main_scope, root, options={}
+        super(main_scope, options)
+        self.root      = File.expand_path(root)
+        self.filter_re = self.options[:filter_re] || /.*/
+      end
+
+      def dirs
+        @dirs ||= Dir[File.join(root, '*')].select{|d| File.directory?(d) }.map { |path| Directory.new(path, filter_re) }
+      end
+
+      def get_metrics metrics, since
+        recent = Directory.recency_filter
+        dirs.each do |dir|
+          metrics << [scope(dir.name, 'num_files', hostname), dir.num_files(&recent)   ]
+          metrics << [scope(dir.name, 'size', hostname),      dir.size(&recent)        ]
+          metrics << [scope(dir.name, 'lines', hostname),     dir.line_counts(&recent) ]
+        end
+      end
+
+      # A class for monitoring the contents of a directory.
+      class Directory
+        
+        # Path to sample for files
+        attr_accessor :path
+        
+        # Wildcard sequence for files under the current directory.
+        attr_accessor :filter_re
+        
+        # A recent file was modified within this window.
+        attr_accessor :recent_window
+        
+        # Only consider the last this-many files.
+        MAX_FILES = 30
+
+        def initialize path, filter_re=/.*/, options={}
+          self.path      = path
+          self.filter_re = filter_re
+        end
+
+        # Name for this pool, suitable for inclusion in a Graphite
+        # target.
+        #
+        # @return [String]
+        def name
+          path.gsub(/\./,'_').gsub(%r{/}, '.').gsub(%r{(^\.|\.$)},'')
+        end
+
+        #
+        # Lists all files in the pool
+        # @param filter_block files only keeps filenames that pass this filter
+        #
+        def files &filter_block
+          Dir[File.join(path, '**/*')].
+            reject{|f| File.directory?(f) }.
+            select{|f| f =~ filter_re }.
+            sort.reverse[0..MAX_FILES].
+            select(&filter_block)
+        end
+
+        def num_files &filter_block
+          files(&filter_block).count
+        end
+
+        def sizes &filter_block
+          files(&filter_block).map{|f| File.size(f) rescue nil }.compact
+        end
+        
+        def size &filter_block
+          sizes(&filter_block).sum
+        end
+        
+        def avg_size &filter_block
+          sizes(&filter_block).sum.to_f / num_files(&filter_block).to_f
+        end
+
+        def lines_in_result_of command, *args
+          begin
+            escaped_args = args.map{|f| "'#{f}'" }
+            result = `#{command} #{escaped_args.join(" ")}`.chomp
+            result.split(/[\r\n]+/)
+          rescue StandardError => e ; warn(e.backtrace, e) ; return nil ; end
+        end
+
+        def wc
+          @wc ||= `which wc`.chomp
+        end
+
+        def line_counts &filter_block
+          files  = files(&filter_block) ; return 0 if files.blank?
+          result = lines_in_result_of(wc, '-l', *files) or return 0
+          counts = result.map{|string| string =~ /^\s*(\d+)\s+/ and $1 }.compact
+          counts.map(&:to_i).sum
+        end
+
+        def self.recent? file
+          (Time.now - File.mtime(file)) < 3600
+        end
+        
+        def self.recency_filter
+          Proc.new{|file| recent?(file) }
+        end
+        
+      end
+    end
+  end
+end
+

data/lib/graphiterb/monitors/disk_space.rb

@@ -0,0 +1,29 @@
+module Graphiterb
+  module Monitors
+
+    # A monitor for how much space is available on a node.
+    class DiskSpace < Graphiterb::Monitors::PeriodicMonitor
+
+      # Runs and parses `df'.
+      #
+      #   disk_space.df
+      #   #=> [["/dev/sda", "39373712", "20488716", "16884908", "55%", "/"], ["/dev/sdb", "920090332", "397413344", "475939088", "46%", "/home"]]
+      #
+      # @return [Array<Array>]
+      def df
+        `/bin/df`.chomp.split("\n").
+          grep(%r{^/dev/}).
+          map{|line| line.split(/\s+/) } rescue []
+      end
+
+      # Calls +df+ and adds the space available metric.
+      def get_metrics metrics, since
+        #         "/dev/sdb1", "39373712", "20488716", "16884908", "55%", "/"
+        df.each do |handle, size, spaceused, spacefree, percentfree, location|
+          disk_name = handle.gsub(/^\//, '').split('/')
+          metrics << [scope(hostname, disk_name, 'available'), spacefree.to_i]
+        end
+      end
+    end
+  end
+end

data/lib/graphiterb/monitors.rb

@@ -0,0 +1,160 @@
+module Graphiterb
+  module Monitors
+
+    autoload :DiskSpace,      'graphiterb/monitors/disk_space'
+    autoload :DirectoryTree,  'graphiterb/monitors/directory_tree'
+
+    # Accepts a lightweight call every iteration.
+    #
+    # Once either a time or an iteration criterion is met, executes the block
+    # and resets the timer until next execution.
+    #
+    # Note that the +time_interval+ is measured *excution to execution* and not
+    # in multiples of iter_interval. Say I set a time_interval of 300s, and
+    # happen to iterate at 297s and 310s after start.  Then the monitor will
+    # execute at 310s, and the next execution will happen on or after 610s.
+    #
+    # Also note that when *either* criterion is met, *both* criteria are
+    # reset. Say I set a time interval of 300s and an +iter_interval+ of 10_000;
+    # and that at 250s I reach iteration 10_000.  Then the monitor will execute
+    # on or after 20_000 iteration or 550s, whichever happens first.
+    #
+    # Stolen from Monkeyshines::Monitor::PeriodicMonitor
+    class PeriodicMonitor
+
+      # The main scope under which the monitor's metrics will be
+      # written.
+      attr_accessor :main_scope
+
+      # Maximum number of seconds that should elapse between running
+      # the monitor.
+      attr_accessor :time_interval
+
+      # Maximum number of internal "iterations" that should elapse between running the monitor
+      attr_accessor :iter_interval
+
+      # The options hash the monitor was created with.
+      attr_accessor :options
+
+      # Internal metrics stored by the monitor.
+      attr_accessor :last_time, :current_iter, :iter, :started_at
+
+      # Provides methods for finding out about the node this code is
+      # running on.
+      include Graphiterb::Utils::SystemInfo
+
+      # Create a new PeriodicMonitor
+      def initialize main_scope, options={}
+        self.main_scope    = main_scope
+        self.started_at    = Time.now.utc.to_f
+        self.last_time     = started_at
+        self.iter          = 0
+        self.current_iter  = 0
+        self.options       = options
+        self.time_interval = options[:time]  || 30 
+        self.iter_interval = options[:iters] || 30 
+      end
+
+      # The Graphiterb::Sender used to communicate with the Graphite
+      # server.
+      #
+      # @return [Graphiterb::Sender]
+      def sender
+        @sender ||= Graphiterb::Sender.new
+      end
+
+      # True if more than +iter_interval+ has elapsed since last
+      # execution.
+      def enough_iterations?
+        iter % iter_interval == 0 if iter_interval
+      end
+
+      # True if more than +time_interval+ has elapsed since last execution.
+      def enough_time? now
+        (now - last_time) > time_interval if time_interval
+      end
+
+      # Time since monitor was created
+      #
+      # @return [Time]
+      def since
+        Time.now.utc.to_f - started_at
+      end
+
+      # Overall iterations per second
+      #
+      # @return [Float]
+      def rate
+        iter.to_f / since.to_f
+      end
+      
+      # "Instantaneous" iterations per second
+      #
+      # @return [Float]
+      def inst_rate now
+        current_iter.to_f / (now-last_time).to_f
+      end
+
+      # Return the scope built from this monitor's +main_scope+ and
+      # the given +names+.
+      #
+      #   monitor.main_scope
+      #   #=> 'system.parameters'
+      #   monitor.scope 'disk', 'space'
+      #   #=> 'system.paramters.disk.space'
+      #
+      # @param [Array<String>] names
+      # @return [String]
+      def scope *names
+        [main_scope, *names].flatten.reject(&:blank?).join('.')
+      end
+
+      # If the interval conditions are met, executes block; otherwise
+      # just does bookkeeping and returns.
+      #
+      # @yield [Array<String>, Time]
+      def periodically &block
+        self.iter += 1
+        self.current_iter += 1
+        now       = Time.now.utc.to_f
+        if enough_iterations? || enough_time?(now)
+          metrics = []
+          block.call(metrics, (now-last_time))
+          sender.send(*metrics)
+          self.last_time = now
+          self.current_iter = 0
+        end
+      end
+
+      # Add metrics to the +metrics+ array.
+      #
+      # This method is meant to be overridden by a sub-class (indeed,
+      # it will raise an error if called directly).
+      #
+      # It should take an array of metrics and a time interval since
+      # last ran and insert metrics into the array.")
+      #
+      # @param [Array<String>] metrics
+      # @param [Time] since the last time the monitor ran
+      def get_metrics metrics, since
+        raise Graphiterb::NotImplementedError.new("Override the get_metrics method of the #{self.class} class")
+      end
+
+      # Run this monitor.
+      #
+      # Sleep for 1 second and then wake up and check if either enough
+      # time (+time_interval+) or enough iterations (+iter_interval+)
+      # have passed run +get_metrics+ if so.
+      def run!
+        loop do
+          periodically do |metrics, since|
+            get_metrics metrics, since
+          end
+          sleep 1
+        end
+      end
+      
+    end
+
+  end
+end

data/lib/graphiterb/{graphite_script.rb → script.rb}

@@ -1,9 +1,6 @@
 require 'rubygems'
 require 'graphiterb'
-require 'wukong/extensions'
-Configliere.use :commandline, :config_file, :define
 
+Configliere.use :commandline, :config_file, :define
 Log = ::Logger.new($stderr) unless defined?(Log)
-
-Settings.read 'graphite.yaml'
 Settings.resolve!

data/lib/graphiterb/{graphite_sender.rb → sender.rb}

@@ -1,23 +1,15 @@
 module Graphiterb
-  #
-  # @example:
-  #   # in initialize, perhaps
-  #   self.sender = GraphiteSender
-  #
-  #   # ... somewhere else ...
-  #   sender.send( ['webserver.pages', pages_this_hour], ['webserver.errors', errors_this_hour] )
-  #
-  class GraphiteSender
+  class Sender
     def initialize
       open!
     end
 
     def open!
       begin
-        Log.warn "Connecting to server #{Settings.carbon_server} port #{Settings.carbon_port}"
+        Graphiterb.log.warn "Connecting to server #{Settings.carbon_server} port #{Settings.carbon_port}"
         @socket = TCPSocket.new(Settings.carbon_server, Settings.carbon_port)
       rescue StandardError => e
-        Log.warn "Couldn't connect to server #{Settings.carbon_server} port #{Settings.carbon_port}: #{e.class} #{e}"
+        Graphiterb.log.warn "Couldn't connect to server #{Settings.carbon_server} port #{Settings.carbon_port}: #{e.class} #{e}"
         $stderr
       end
     end
@@ -30,7 +22,7 @@ module Graphiterb
       begin
         block.call
       rescue StandardError => e
-        Log.warn "Sleeping #{Settings.on_error_delay}: #{e.class} #{e}"
+        Graphiterb.log.warn "Sleeping #{Settings.on_error_delay}: #{e.class} #{e}"
         sleep Settings.on_error_delay
         @socket = nil
         return nil
@@ -46,7 +38,7 @@ module Graphiterb
       safely do
         message = metrics.map{|metric, val, ts| [metric, val, (ts||now)].join(" ") }.join("\n")+"\n"
         socket.puts(message)
-        Log.info message.gsub(/\n+/, "\t")
+        Graphiterb.log.info message.gsub(/\n+/, "\t")
       end
     end
   end

data/lib/graphiterb/utils/extensions.rb

@@ -0,0 +1,7 @@
+class Array
+
+  def sum
+    inject(0.0) { |partial_sum, element| partial_sum += element }
+  end
+  
+end

data/lib/graphiterb/utils/log.rb

@@ -0,0 +1,16 @@
+require 'logger'
+
+module Graphiterb
+
+  class << self; attr_accessor :log end
+
+  # Create a Logger and point it at Graphiterb::LOG_FILE_DESTINATION which is
+  # set in ~/.imwrc and defaults to STDERR.
+  def self.instantiate_logger!
+    Graphiterb.log ||= Logger.new(Settings[:log])
+    Graphiterb.log.datetime_format = "%Y%m%d-%H:%M:%S "
+    Graphiterb.log.level           = Logger::INFO
+  end
+
+end
+Graphiterb.instantiate_logger!

data/lib/graphiterb/utils/system.rb

@@ -0,0 +1,17 @@
+module Graphiterb
+  module Utils
+
+    # A module which provides information about the node this code is
+    # executing on.
+    #
+    # Maybe it's worth bringing Ohai into this.  I'm not sure.
+    module SystemInfo
+
+      def hostname
+        @hostname ||= `hostname`.chomp.gsub(".","_")
+      end
+      
+    end
+  end
+end
+

data/lib/graphiterb/utils.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'socket'
+require 'configliere'
+
+require 'graphiterb/utils/log'
+require 'graphiterb/utils/extensions'
+
+module Graphiterb
+
+  Error               = Class.new(StandardError)
+  NotImplementedError = Class.new(Error)
+  
+  module Utils
+    autoload :SystemInfo, 'graphiterb/utils/system'
+  end
+end
+

data/lib/graphiterb.rb

@@ -1,15 +1,16 @@
-require 'socket'
-require 'logger'
+require 'graphiterb/utils'
 
-require 'configliere'
-Configliere.use :define
+Settings.use :define, :config_file
+
+Settings.define :log,            :default => STDOUT,      :description => "Log output for Graphiterb"
 Settings.define :carbon_server,  :default => 'localhost', :description => "Host address for carbon database server", :required => true
 Settings.define :carbon_port,    :default => '2003',      :description => "Port for carbon database server", :required => true
 Settings.define :update_delay,   :default => 30,          :description => "How long to wait between updates. Must be faster than the value in the graphite/conf/storage-schemas", :required => true, :type => Integer
 Settings.define :on_error_delay, :default => 0.1,         :description => "How long to wait on connect errors", :required => true, :type => Float
 
-require 'graphiterb/graphite_sender'
+Settings.read '/etc/graphiterb/graphiterb.yaml' if File.exist? '/etc/graphiterb/graphiterb.yaml'
+
 module Graphiterb
-  autoload :GraphiteLogger,       'graphiterb/graphite_logger'
-  autoload :GraphiteSystemLogger, 'graphiterb/graphite_system_logger'
+  autoload :Monitors, 'graphiterb/monitors'
+  autoload :Sender,   'graphiterb/sender'
 end

data/spec/graphiterb_spec.rb

@@ -2,6 +2,6 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "Graphiterb" do
   it "fails" do
-    fail "hey buddy, you should probably rename this file and start specing for real"
+    fail "hey buddy, you should probably rename this file and start specing for real.  seriously."
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: graphiterb
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 5
-  version: 0.1.5
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Philip (flip) Kromer (@mrflip)
@@ -15,13 +15,27 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-16 00:00:00 +00:00
+date: 2010-08-17 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: configliere
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -33,11 +47,11 @@ dependencies:
         - 9
         version: 1.2.9
   type: :development
-  version_requirements: *id001
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: yard
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -47,7 +61,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id002
+  version_requirements: *id003
 description: Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com
 email: info@infochimps.org
 executables: []
@@ -73,10 +87,15 @@ files:
 - examples/toy.rb
 - graphiterb.gemspec
 - lib/graphiterb.rb
-- lib/graphiterb/graphite_logger.rb
-- lib/graphiterb/graphite_script.rb
-- lib/graphiterb/graphite_sender.rb
-- lib/graphiterb/graphite_system_logger.rb
+- lib/graphiterb/monitors.rb
+- lib/graphiterb/monitors/directory_tree.rb
+- lib/graphiterb/monitors/disk_space.rb
+- lib/graphiterb/script.rb
+- lib/graphiterb/sender.rb
+- lib/graphiterb/utils.rb
+- lib/graphiterb/utils/extensions.rb
+- lib/graphiterb/utils/log.rb
+- lib/graphiterb/utils/system.rb
 - spec/graphiterb_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
@@ -117,8 +136,8 @@ summary: Fast Ubiquitous Dashboard Logs with Graphite (http://graphite.wikidot.c
 test_files: 
 - spec/graphiterb_spec.rb
 - spec/spec_helper.rb
+- examples/toy.rb
 - examples/storage_monitor.rb
-- examples/api_call_monitor.rb
 - examples/loadavg_graphite_sender.rb
-- examples/toy.rb
 - examples/file_monitor.rb
+- examples/api_call_monitor.rb

data/lib/graphiterb/graphite_logger.rb

@@ -1,42 +0,0 @@
-require 'monkeyshines/monitor'
-
-module Graphiterb
-  class GraphiteLogger < Monkeyshines::Monitor::PeriodicMonitor
-    # Connection to graphite server
-    attr_reader :sender
-    # the leading segment for sent metrics -- eg 'scrapers' or 'api_calls'
-    attr_reader :main_scope
-
-    def initialize main_scope, *args
-      super *args
-      self.time_interval ||= Settings.update_delay
-      @sender     = GraphiteSender.new
-      @main_scope = main_scope
-    end
-
-    def periodically &block
-      super do |iter, since|
-        metrics = []
-        block.call(metrics, iter, since)
-        sender.send *metrics
-      end
-    end
-
-    def hostname
-      @host ||= `hostname`.chomp.gsub(".","_")
-    end
-
-    def scope_name *scope
-      [main_scope, scope].flatten.reject(&:blank?).join('.')
-    end
-
-    def run!
-      loop do
-        periodically do |metrics, iter, since|
-          get_metrics metrics, iter, since
-        end
-        sleep 1
-      end
-    end
-  end
-end

data/lib/graphiterb/graphite_system_logger.rb

@@ -1,5 +0,0 @@
-module Graphiterb
-  class GraphiteSystemLogger < GraphiteLogger
-
-  end
-end