graphiterb 0.2.11 → 0.2.12

data/.gitignore

@@ -1,21 +1,22 @@
-## MAC OS
-.DS_Store
 
-## TEXTMATE
-*.tmproj
-tmtags
 
-## EMACS
-*~
-\#*
-.\#*
 
-## VIM
-*.swp
 
+
+## EMACS
+## MAC OS
 ## PROJECT::GENERAL
+## PROJECT::SPECIFIC
+## TEXTMATE
+## VIM
+*.swp
+*.tmproj
+*~
+.DS_Store
+.\#*
+/TAGS
+\#*
 coverage
-rdoc
 pkg
-
-## PROJECT::SPECIFIC
+rdoc
+tmtags

data/VERSION

@@ -1 +1 @@
-0.2.11
+0.2.12

data/examples/toy.rb

@@ -3,24 +3,12 @@ require 'rubygems'
 $: <<  File.dirname(__FILE__)+'/../lib'
 require 'graphiterb'
 Configliere.use :commandline, :config_file
-
-Settings.read 'graphite.yaml'
 Settings.resolve!
-Log = Logger.new($stderr) unless defined?(Log)
-
-monitor = Graphiterb::GraphiteLogger.new(:iters => nil, :time => 5)
-
-handle = 'simple_toy'
 
-loop do
-  monitor.periodically do |metrics, iter, since|
-    metrics << ["scraper.toy.#{handle}.iter", iter]
-    metrics << ["scraper.toy.#{handle}.iter", iter]
-    metrics << ["scraper.toy.#{handle}.iter", iter]
-    metrics << ["scraper.toy.#{handle}.iter", iter]
+class ToyMonitor < Graphiterb::Monitors::PeriodicMonitor
+  def get_metrics metrics, since
+    metrics << [scope('random', graphite_identifier), rand]
   end
-  delay = 2
-  sleep delay
-  print delay.to_s+"\t"
 end
 
+ToyMonitor.new('toy').run! if $0 == __FILE__

data/graphiterb.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{graphiterb}
-  s.version = "0.2.11"
+  s.version = "0.2.12"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer (@mrflip)"]
-  s.date = %q{2010-09-22}
+  s.date = %q{2010-09-28}
   s.description = %q{Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com}
   s.email = %q{info@infochimps.org}
   s.extra_rdoc_files = [
@@ -24,11 +24,6 @@ Gem::Specification.new do |s|
      "README.textile",
      "Rakefile",
      "VERSION",
-     "examples/api_call_monitor.rb",
-     "examples/file_monitor.rb",
-     "examples/loadavg_graphite_sender.rb",
-     "examples/run_servers.sh",
-     "examples/storage_monitor.rb",
      "examples/toy.rb",
      "graphiterb.gemspec",
      "lib/graphiterb.rb",
@@ -55,11 +50,7 @@ Gem::Specification.new do |s|
   s.test_files = [
     "spec/graphiterb_spec.rb",
      "spec/spec_helper.rb",
-     "examples/toy.rb",
-     "examples/storage_monitor.rb",
-     "examples/loadavg_graphite_sender.rb",
-     "examples/file_monitor.rb",
-     "examples/api_call_monitor.rb"
+     "examples/toy.rb"
   ]
 
   if s.respond_to? :specification_version then

data/lib/graphiterb.rb

@@ -13,5 +13,6 @@ Settings.define :carbon_server,  :default => 'localhost', :description => "Host
 Settings.define :carbon_port,    :default => '2003',      :description => "Port for carbon database server", :required => true
 Settings.define :update_delay,   :default => 30,          :description => "How long to wait between updates. Must be faster than the value in the graphite/conf/storage-schemas", :required => true, :type => Integer
 Settings.define :on_error_delay, :default => 0.1,         :description => "How long to wait on connect errors", :required => true, :type => Float
-
+Settings.define :node_name_file, :default => '/etc/node_name', :description => "Path to a file which contains the name of this node", :type => String
 Settings.read '/etc/graphiterb/graphiterb.yaml' if File.exist? '/etc/graphiterb/graphiterb.yaml'
+Settings.resolve!

data/lib/graphiterb/utils/system.rb

@@ -8,9 +8,16 @@ module Graphiterb
     module SystemInfo
 
       def hostname
-        @hostname ||= `hostname`.chomp.gsub(".","_")
+        @hostname ||= `hostname`.chomp.gsub(/\./,"_")
+      end
+
+      def node_name
+        @node_name ||= Settings[:node_name_file] && File.exist?(Settings[:node_name_file]) && File.read(Settings[:node_name_file]).chomp.strip.gsub(/\./, '_')
+      end
+
+      def graphite_identifier
+        node_name || hostname
       end
-      
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: graphiterb
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 15
   prerelease: false
   segments: 
   - 0
   - 2
-  - 11
-  version: 0.2.11
+  - 12
+  version: 0.2.12
 platform: ruby
 authors: 
 - Philip (flip) Kromer (@mrflip)
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-22 00:00:00 -05:00
+date: 2010-09-28 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -79,11 +79,6 @@ files:
 - README.textile
 - Rakefile
 - VERSION
-- examples/api_call_monitor.rb
-- examples/file_monitor.rb
-- examples/loadavg_graphite_sender.rb
-- examples/run_servers.sh
-- examples/storage_monitor.rb
 - examples/toy.rb
 - graphiterb.gemspec
 - lib/graphiterb.rb
@@ -139,7 +134,3 @@ test_files:
 - spec/graphiterb_spec.rb
 - spec/spec_helper.rb
 - examples/toy.rb
-- examples/storage_monitor.rb
-- examples/loadavg_graphite_sender.rb
-- examples/file_monitor.rb
-- examples/api_call_monitor.rb

data/examples/api_call_monitor.rb

@@ -1,51 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'graphiterb'
-require 'graphiterb/graphite_script'
-
-WC_EXEC = '/usr/bin/wc'
-
-class ApiCallMonitor < Graphiterb::GraphiteLogger
-  API_CALLS_TO_MONITOR   = %w[trstrank wordbag influence conversation]
-  ERROR_CODES_TO_MONITOR = %w[4.. 5.. 200]
-
-  def initialize *args
-    super *args
-    @current_total = Hash.new
-    @prev_total    = Hash.new
-  end
-
-  def calls api
-    total_calls = `cat /var/www/apeyeye/shared/log/apeyeye-access.log | egrep 'GET /soc/net/tw/#{api}' | #{WC_EXEC} -l` rescue 0
-    @current_total[api] = total_calls.to_i
-  end
-
-  def errors error_code
-    log_cat =  `cat /var/www/apeyeye/shared/log/apeyeye-access.log | egrep 'GET /soc/net/tw/.*HTTP/1\.[0-1]..#{error_code}' | #{WC_EXEC} -l` rescue 0
-    @current_total[error_code] = log_cat.to_i
-  end
-
-  def rate item
-    @prev_total[item] ||= @current_total[item]
-    rate                = @current_total[item].to_i - @prev_total[item].to_i
-    @prev_total[item]   = @current_total[item]
-    [0, rate].max
-  end
-
-  def get_metrics metrics, iter, since
-    API_CALLS_TO_MONITOR.each do |api|
-      metrics << [scope_name(hostname, api, 'total_accesses'), calls(api)]
-      metrics << [scope_name(hostname, api, 'accesses'),       rate(api)]
-    end
-    ERROR_CODES_TO_MONITOR.each do |code|
-      metrics << [scope_name(hostname, code.gsub('.','x'), 'total_errors'), errors(code)]
-      metrics << [scope_name(hostname, code.gsub('.','x'), 'errors'),       rate(code)]
-    end
-  end
-end
-
-
-warn "Update delay is #{Settings.update_delay} seconds.  You probably want something larger: some of these checks are data-intensive" if Settings.update_delay < 60
-Settings.die "Update delay is #{Settings.update_delay} seconds.  You need to radio in at least as often as /usr/local/share/graphite/conf/storage-schemas says -- this is typically 5 minutes." if Settings.update_delay >= 300
-
-ApiCallMonitor.new('apeyeye', :iters => nil, :time => Settings.update_delay).run!

data/examples/file_monitor.rb

@@ -1,113 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'graphiterb'
-Settings.define :work_dir, :description => "Base directory where scrapers store files. (Ex: /data/ripd/com.tw)", :required => true
-require 'graphiterb/graphite_script'
-
-#
-# Usage:
-#
-#    nohup ~/ics/backend/graphiterb/bin/file_monitor.rb --work_dir=/data/ripd/com.tw --carbon_server=whatever --update_delay=120 > /data/log/file_monitor.log 2>&1 &
-#
-
-WC_EXEC = '/usr/bin/wc'
-
-class FilePool
-  # Path to sample for files
-  attr_accessor :path
-  # wildcard sequence for files under the current directory
-  attr_accessor :filter_re
-  # A recent file was modified within this window
-  attr_accessor :recent_window
-  # Only consider the last this-many files
-  MAX_FILES = 30
-
-  def initialize path, filter_re=/.*/, options={}
-    self.path      = path
-    self.filter_re = filter_re
-  end
-
-  # Name for this pool, suitable for inclusion in a metrics handle
-  def name
-    path.gsub(/\./,'_').gsub(%r{/}, '.').gsub(%r{(^\.|\.$)},'')
-  end
-
-  #
-  # Lists all files in the pool
-  # @param filter_block files only keeps filenames that pass this filter
-  #
-  def files &filter_block
-    Dir[File.join(path, '**/*')].
-      reject{|f| File.directory?(f) }.
-      select{|f| f =~ filter_re }.
-      sort.reverse[0..MAX_FILES].
-      select(&filter_block)
-  end
-
-  def num_files &filter_block
-    files(&filter_block).count
-  end
-
-  def sizes &filter_block
-    files(&filter_block).map{|f| File.size(f) rescue nil }.compact
-  end
-  def size &filter_block
-    sizes(&filter_block).sum
-  end
-  def avg_size &filter_block
-    sizes(&filter_block).sum.to_f / num_files(&filter_block).to_f
-  end
-
-  def lines_in_result_of command, *args
-    begin
-      escaped_args = args.map{|f| "'#{f}'" }
-      result = `#{command} #{escaped_args.join(" ")}`.chomp
-      result.split(/[\r\n]+/)
-    rescue StandardError => e ; warn(e.backtrace, e) ; return nil ; end
-  end
-
-  def line_counts &filter_block
-    files  = files(&filter_block) ; return 0 if files.blank?
-    result = lines_in_result_of(WC_EXEC, '-l', *files) or return 0
-    counts = result.map{|wc| wc =~ /^\s*(\d+)\s+/ and $1 }.compact
-    counts.map(&:to_i).sum
-  end
-
-  def self.recent? file
-    (Time.now - File.mtime(file)) < 3600
-  end
-  def self.recency_filter
-    Proc.new{|file| recent?(file) }
-  end
-end
-
-class FileMonitor < Graphiterb::GraphiteSystemLogger
-  attr_accessor :path
-  attr_accessor :pools
-
-  def initialize *args
-    super *args
-    self.path = Settings.work_dir
-    self.pools = {}
-    populate_pools!
-  end
-
-  def populate_pools!
-    Dir[File.join(path, '*')].select{|d| File.directory?(d) }.each do |dir|
-      self.pools[dir] ||= FilePool.new(dir, %r{20\d*/.*\.(?:tsv|json|xml)})
-    end
-  end
-
-  def get_metrics metrics, iter, since
-    recent = FilePool.recency_filter
-    pools.each do |pool_path, pool|
-      metrics << [scope_name(pool.name, hostname, 'active_files'),     pool.num_files(&recent)   ]
-      metrics << [scope_name(pool.name, hostname, 'active_file_size'), pool.size(&recent)        ]
-      metrics << [scope_name(pool.name, hostname, 'line_counts'),      pool.line_counts(&recent) ]
-    end
-  end
-end
-
-warn "Update delay is #{Settings.update_delay} seconds.  You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 60
-warn "Update delay is #{Settings.update_delay} seconds.  You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 300
-FileMonitor.new('scraper', :iters => nil, :time => Settings.update_delay).run!

data/examples/loadavg_graphite_sender.rb

@@ -1,36 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-$: <<  File.dirname(__FILE__)+'/../lib'
-require 'graphiterb'
-Configliere.use :commandline, :config_file
-
-Settings.read 'graphite.yaml'
-Settings.resolve!
-Log = Logger.new($stderr) unless defined?(Log)
-
-class LoadavgGraphiteSender < Graphiterb::GraphiteSender
-  def hostname
-    @hostname ||= `hostname`.chomp
-  end
-
-  def loadavgs
-    # File.open('/proc/loadavg').read.strip.split[0..2]
-    `uptime`.chomp.gsub(/.*:\s+/, '').split(/[,\s]+/)
-  end
-
-  def loadavgs_metrics
-    %w[1min 5min 15min].zip(loadavgs).map do |duration, avg|
-      ["system.#{hostname}.loadavg_#{duration}", avg]
-    end
-  end
-
-  def send_loop
-    loop do
-      send *loadavgs_metrics
-      Log.info "Sleeping #{Settings.update_delay}"
-      sleep Settings.update_delay.to_i
-    end
-  end
-end
-
-LoadavgGraphiteSender.new.send_loop

data/examples/run_servers.sh

@@ -1,2 +0,0 @@
-PYTHONPATH=$pwd/whisper nohup ./bin/run-graphite-devel-server.py --libs=$pwd/webapp/ /usr/local/share/graphite/ >> ./storage/log/webapp/server.log 2>&1 &
-PYTHONPATH=$pwd/whisper nohup ./carbon/bin/carbon-cache.py --debug start >> ./storage/log/carbon-cache/console.log 2>&1 &

data/examples/storage_monitor.rb

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'rubygems'
-require 'graphiterb/graphite_script'
-
-class AvailSpaceMonitor < Graphiterb::GraphiteLogger
-  def diskfree
-    `/bin/df`.chomp.split("\n").
-      grep(%r{^/dev/}).
-      map{|line| line.split(/\s+/) } rescue []
-  end
-
-  def get_metrics metrics, iter, since
-    diskfree.each do |handle, size, spaceused, spacefree, percentfree, location|
-      metrics << ["system.#{hostname}#{handle.gsub(/\//,'.')}.available", spacefree.to_i]
-    end
-  end
-end
-
-warn "Update delay is #{Settings.update_delay} seconds.  You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 30
-warn "Update delay is #{Settings.update_delay} seconds.  You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 60
-
-AvailSpaceMonitor.new('system').run!