graphiterb 0.0.1 → 0.1.0

data/README.textile

@@ -0,0 +1,35 @@
+= graphiterb
+
+Fast, Ubiquitous dashboard logging with Graphite.
+
+<pre>
+    # in initialize, perhaps
+    self.sender = Graphiterb::GraphiteSender
+   
+    # ... somewhere else ...
+    sender.send( ['webserver.pages', pages_this_hour], ['webserver.errors', errors_this_hour] )
+</pre>
+
+... and also see the examples, like bin/loadavg_graphite_sender.rb
+
+Uses:
+* "Graphite":http://graphite.wikidot.com
+
+Depends on:
+* "Configliere":http://github.com/mrflip/configliere
+
+h2. Other
+
+h3. Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+h3. Copyright
+
+Copyright (c) 2010 Infochimps (@infochimps). See LICENSE for details.

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.1.0

data/bin/api_call_monitor.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib/'
+require 'graphite'
+require 'graphite/graphite_script'
+
+WC_EXEC = '/usr/bin/wc'
+
+class ApiCallMonitor < Graphiterb::GraphiteLogger
+  API_CALLS_TO_MONITOR   = %w(trstrank wordbag influence conversation)
+  ERROR_CODES_TO_MONITOR = %w(4.. 5..)
+
+  def initialize *args
+    super *args
+    @current_total = Hash.new
+    @prev_total    = Hash.new
+  end
+
+  def calls api
+    total_calls = `cat /var/www/apeyeye/shared/log/apeyeye-access.log | grep 'GET /soc/net/tw/#{api}' | #{WC_EXEC} -l`
+    @current_total[api] = total_calls
+  end
+
+  def errors error_code
+    log_cat =  `cat /var/www/apeyeye/shared/log/apeyeye-access.log | grep 'GET /soc/net/tw/.*HTTP/1\.[0-1]..#{error_code}' | #{WC_EXEC} -l`
+    @current_total[error_code] = log_cat
+  end
+
+  def rate item
+    @prev_total[item] ||= @current_total[item]
+    rate                = @current_total[item].to_i - @prev_total[item].to_i
+    @prev_total[item]   = @current_total[item]
+    [0, rate].max
+  end
+
+  def hostname
+    @hostname ||= `hostname`.chomp.gsub(".","_")
+  end
+
+  def get_metrics metrics, iter, since
+    API_CALLS_TO_MONITOR.each do |api|
+      calls(api)
+      metrics << [scope_name(hostname, api, 'accesses'), rate(api)]
+    end
+    ERROR_CODES_TO_MONITOR.each do |code|
+      errors(code)
+      metrics << [scope_name(hostname, code.gsub('.','x'), 'errors'), rate(code)]
+    end
+  end
+end
+
+
+warn "Update delay is #{Settings.update_delay} seconds.  You probably want something larger: some of these checks are data-intensive" if Settings.update_delay < 60
+Settings.die "Update delay is #{Settings.update_delay} seconds.  You need to radio in at least as often as /usr/local/share/graphite/conf/storage-schemas says -- this is typically 5 minutes." if Settings.update_delay >= 300
+
+ApiCallMonitor.new('apeyeye', :iters => nil, :time => Settings.update_delay).run!

data/bin/file_monitor.rb

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib/'
+require 'graphite'
+Settings.define :work_dir, :description => "Base directory where scrapers store files. (Ex: /data/ripd/com.tw)", :required => true
+require 'graphite/graphite_script'
+
+#
+# Usage:
+#
+#    nohup ~/ics/backend/graphiterb/bin/file_monitor.rb --work_dir=/data/ripd/com.tw --carbon_server=whatever --update_delay=120 > /data/log/file_monitor.log 2>&1 &
+#
+
+WC_EXEC = '/usr/bin/wc'
+
+class FilePool
+  # Path to sample for files
+  attr_accessor :path
+  # wildcard sequence for files under the current directory
+  attr_accessor :filter_re
+  # A recent file was modified within this window
+  attr_accessor :recent_window
+  # Only consider the last this-many files
+  MAX_FILES = 30
+
+  def initialize path, filter_re=/.*/, options={}
+    self.path      = path
+    self.filter_re = filter_re
+  end
+
+  # Name for this pool, suitable for inclusion in a metrics handle
+  def name
+    path.gsub(/\./,'_').gsub(%r{/}, '.').gsub(%r{(^\.|\.$)},'')
+  end
+
+  #
+  # Lists all files in the pool
+  # @param filter_block files only keeps filenames that pass this filter
+  #
+  def files &filter_block
+    Dir[File.join(path, '**/*')].
+      reject{|f| File.directory?(f) }.
+      select{|f| f =~ filter_re }.
+      sort.reverse[0..MAX_FILES].
+      select(&filter_block)
+  end
+
+  def num_files &filter_block
+    files(&filter_block).count
+  end
+
+  def sizes &filter_block
+    files(&filter_block).map{|f| File.size(f) rescue nil }.compact
+  end
+  def size &filter_block
+    sizes(&filter_block).sum
+  end
+  def avg_size &filter_block
+    sizes(&filter_block).sum.to_f / num_files(&filter_block).to_f
+  end
+
+  def lines_in_result_of command, *args
+    begin
+      escaped_args = args.map{|f| "'#{f}'" }
+      result = `#{command} #{escaped_args.join(" ")}`.chomp
+      result.split(/[\r\n]+/)
+    rescue StandardError => e ; warn(e.backtrace, e) ; return nil ; end
+  end
+
+  def line_counts &filter_block
+    files  = files(&filter_block) ; return 0 if files.blank?
+    result = lines_in_result_of(WC_EXEC, '-l', *files) or return 0
+    counts = result.map{|wc| wc =~ /^\s*(\d+)\s+/ and $1 }.compact
+    counts.map(&:to_i).sum
+  end
+
+  def self.recent? file
+    (Time.now - File.mtime(file)) < 1.hour
+  end
+  def self.recency_filter
+    Proc.new{|file| recent?(file) }
+  end
+end
+
+class FileMonitor < Graphiterb::GraphiteSystemLogger
+  attr_accessor :path
+  attr_accessor :pools
+
+  def initialize *args
+    super *args
+    self.path = Settings.work_dir
+    self.pools = {}
+    populate_pools!
+  end
+
+  def populate_pools!
+    Dir[File.join(path, '*')].select{|d| File.directory?(d) }.each do |dir|
+      self.pools[dir] ||= FilePool.new(dir, %r{20\d*/.*\.(?:tsv|json|xml)})
+    end
+  end
+
+  def get_metrics metrics, iter, since
+    recent = FilePool.recency_filter
+    pools.each do |pool_path, pool|
+      metrics << [scope_name(pool.name, hostname, 'active_files'),     pool.num_files(&recent)   ]
+      metrics << [scope_name(pool.name, hostname, 'active_file_size'), pool.size(&recent)        ]
+      metrics << [scope_name(pool.name, hostname, 'line_counts'),      pool.line_counts(&recent) ]
+    end
+  end
+end
+
+warn "Update delay is #{Settings.update_delay} seconds.  You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 60
+warn "Update delay is #{Settings.update_delay} seconds.  You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 300
+FileMonitor.new('scraper', :iters => nil, :time => Settings.update_delay).run!

data/bin/{example-client.rb → loadavg_graphite_sender.rb}

@@ -8,14 +8,19 @@ Settings.read 'graphite.yaml'
 Settings.resolve!
 Log = Logger.new($stderr) unless defined?(Log)
 
-class LoadavgGraphiteSender < GraphiteRb::GraphiteSender
+class LoadavgGraphiteSender < Graphiterb::GraphiteSender
+  def hostname
+    @hostname ||= `hostname`.chomp
+  end
+
   def loadavgs
-    File.open('/proc/loadavg').read.strip.split[0..2]
+    # File.open('/proc/loadavg').read.strip.split[0..2]
+    `uptime`.chomp.gsub(/.*:\s+/, '').split(/[,\s]+/)
   end
 
-  def loadavgs_metrics 
+  def loadavgs_metrics
     %w[1min 5min 15min].zip(loadavgs).map do |duration, avg|
-      ["system_rb.loadavg_#{duration}", avg]
+      ["system.#{hostname}.loadavg_#{duration}", avg]
     end
   end
 

data/bin/run_servers.sh

@@ -0,0 +1,2 @@
+PYTHONPATH=$pwd/whisper nohup ./bin/run-graphite-devel-server.py --libs=$pwd/webapp/ /usr/local/share/graphite/ >> ./storage/log/webapp/server.log 2>&1 &
+PYTHONPATH=$pwd/whisper nohup ./carbon/bin/carbon-cache.py --debug start >> ./storage/log/carbon-cache/console.log 2>&1 &

data/bin/storage_monitor.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib/'
+require 'rubygems'
+require 'graphiterb/graphite_script'
+
+class AvailSpaceMonitor < Graphiterb::GraphiteLogger
+  def diskfree
+    `/bin/df`.chomp.split("\n").
+      grep(%r{^/dev/}).
+      map{|line| line.split(/\s+/) } rescue []
+  end
+
+  def get_metrics metrics, iter, since
+    diskfree.each do |handle, size, spaceused, spacefree, percentfree, location|
+      metrics << ["system.#{hostname}#{handle.gsub(/\//,'.')}.available", spacefree.to_i]
+    end
+  end
+end
+
+warn "Update delay is #{Settings.update_delay} seconds.  You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 30
+warn "Update delay is #{Settings.update_delay} seconds.  You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 60
+
+AvailSpaceMonitor.new('system').run!

data/bin/toy.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+$: <<  File.dirname(__FILE__)+'/../lib'
+require 'graphiterb'
+Configliere.use :commandline, :config_file
+
+Settings.read 'graphite.yaml'
+Settings.resolve!
+Log = Logger.new($stderr) unless defined?(Log)
+
+monitor = Graphiterb::GraphiteLogger.new(:iters => nil, :time => 5)
+
+handle = 'simple_toy'
+
+loop do
+  monitor.periodically do |metrics, iter, since|
+    metrics << ["scraper.toy.#{handle}.iter", iter]
+    metrics << ["scraper.toy.#{handle}.iter", iter]
+    metrics << ["scraper.toy.#{handle}.iter", iter]
+    metrics << ["scraper.toy.#{handle}.iter", iter]
+  end
+  delay = 2
+  sleep delay
+  print delay.to_s+"\t"
+end
+

data/graphiterb.gemspec

@@ -5,31 +5,38 @@
 
 Gem::Specification.new do |s|
   s.name = %q{graphiterb}
-  s.version = "0.0.1"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer (@mrflip)"]
-  s.date = %q{2010-03-02}
-  s.default_executable = %q{example-client.rb}
+  s.date = %q{2010-07-19}
   s.description = %q{Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com}
   s.email = %q{info@infochimps.org}
-  s.executables = ["example-client.rb"]
+  s.executables = ["run_servers.sh", "storage_monitor.rb", "api_call_monitor.rb", "loadavg_graphite_sender.rb", "toy.rb", "file_monitor.rb"]
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.rdoc"
+     "README.textile"
   ]
   s.files = [
     ".document",
      ".gitignore",
      "CHANGELOG",
      "LICENSE",
-     "README.rdoc",
+     "README.textile",
      "Rakefile",
      "VERSION",
-     "bin/example-client.rb",
+     "bin/api_call_monitor.rb",
+     "bin/file_monitor.rb",
+     "bin/loadavg_graphite_sender.rb",
+     "bin/run_servers.sh",
+     "bin/storage_monitor.rb",
+     "bin/toy.rb",
      "graphiterb.gemspec",
      "lib/graphiterb.rb",
+     "lib/graphiterb/graphite_logger.rb",
+     "lib/graphiterb/graphite_script.rb",
      "lib/graphiterb/graphite_sender.rb",
+     "lib/graphiterb/graphite_system_logger.rb",
      "spec/graphiterb_spec.rb",
      "spec/spec.opts",
      "spec/spec_helper.rb"
@@ -37,18 +44,18 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/mrflip/graphiterb}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Fast Ubiquitous Dashboard Logs with Graphite (http://graphite.wikidot.com)}
   s.test_files = [
-    "spec/spec_helper.rb",
-     "spec/graphiterb_spec.rb"
+    "spec/graphiterb_spec.rb",
+     "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_development_dependency(%q<yard>, [">= 0"])
     else

data/lib/graphiterb/graphite_logger.rb

@@ -0,0 +1,42 @@
+require 'monkeyshines/monitor'
+
+module Graphiterb
+  class GraphiteLogger < Monkeyshines::Monitor::PeriodicMonitor
+    # Connection to graphite server
+    attr_reader :sender
+    # the leading segment for sent metrics -- eg 'scrapers' or 'api_calls'
+    attr_reader :main_scope
+
+    def initialize main_scope, *args
+      super *args
+      self.time_interval ||= Settings.update_delay
+      @sender     = GraphiteSender.new
+      @main_scope = main_scope
+    end
+
+    def periodically &block
+      super do |iter, since|
+        metrics = []
+        block.call(metrics, iter, since)
+        sender.send *metrics
+      end
+    end
+
+    def hostname
+      @host ||= `hostname`.chomp.gsub(".","_")
+    end
+
+    def scope_name *scope
+      [main_scope, scope].flatten.reject(&:blank?).join('.')
+    end
+
+    def run!
+      loop do
+        periodically do |metrics, iter, since|
+          get_metrics metrics, iter, since
+        end
+        sleep 1
+      end
+    end
+  end
+end

data/lib/graphiterb/graphite_script.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'graphiterb'
+require 'wukong/extensions'
+Configliere.use :commandline, :config_file, :define
+
+Log = ::Logger.new($stderr) unless defined?(Log)
+
+Settings.read 'graphite.yaml'
+Settings.resolve!

data/lib/graphiterb/graphite_sender.rb

@@ -1,4 +1,4 @@
-module GraphiteRb
+module Graphiterb
   #
   # @example:
   #   # in initialize, perhaps
@@ -8,28 +8,30 @@ module GraphiteRb
   #   sender.send( ['webserver.pages', pages_this_hour], ['webserver.errors', errors_this_hour] )
   #
   class GraphiteSender
-    attr_accessor :socket
-
     def initialize
       open!
     end
 
     def open!
       begin
-        warn "Connecting to server #{Settings.carbon_server} port #{Settings.carbon_port}"
-        self.socket = TCPSocket.new(Settings.carbon_server, Settings.carbon_port)
+        Log.warn "Connecting to server #{Settings.carbon_server} port #{Settings.carbon_port}"
+        @socket = TCPSocket.new(Settings.carbon_server, Settings.carbon_port)
       rescue StandardError => e
         Log.warn "Couldn't connect to server #{Settings.carbon_server} port #{Settings.carbon_port}: #{e.class} #{e}"
-        raise e
       end
     end
 
+    def socket
+      @socket ||= open!
+    end
+
     def safely &block
       begin
         block.call
       rescue StandardError => e
         Log.warn "Sleeping #{Settings.on_error_delay}: #{e.class} #{e}"
         sleep Settings.on_error_delay
+        @socket = nil
         return nil
       end
     end
@@ -40,10 +42,10 @@ module GraphiteRb
 
     def send *metrics
       now = timestamp
-      message = metrics.map{|metric, val| [metric, val, now].join(" ") }.join("\n")
+      message = metrics.map{|metric, val, ts| [metric, val, (ts||now)].join(" ") }.join("\n")
       safely do
         socket.puts(message)
-        Log.debug "Sent:\n#{message}\n#{'-'*80}"
+        Log.info message.gsub(/\n+/, "\t")
       end
     end
   end

data/lib/graphiterb/graphite_system_logger.rb

@@ -0,0 +1,5 @@
+module Graphiterb
+  class GraphiteSystemLogger < GraphiteLogger
+
+  end
+end

data/lib/graphiterb.rb

@@ -5,7 +5,11 @@ require 'configliere'
 Configliere.use :define
 Settings.define :carbon_server,  :default => 'localhost', :description => "Host address for carbon database server", :required => true
 Settings.define :carbon_port,    :default => '2003',      :description => "Port for carbon database server", :required => true
-Settings.define :update_delay,   :default => 60,          :description => "How long to wait between updates", :required => true, :type => Integer
-Settings.define :on_error_delay, :default => 60,         :description => "How long to wait on connect errors", :required => true, :type => Integer
+Settings.define :update_delay,   :default => 30,          :description => "How long to wait between updates. Must be faster than the value in the graphite/conf/storage-schemas", :required => true, :type => Integer
+Settings.define :on_error_delay, :default => 0.1,         :description => "How long to wait on connect errors", :required => true, :type => Float
 
 require 'graphiterb/graphite_sender'
+module Graphiterb
+  autoload :GraphiteLogger,       'graphiterb/graphite_logger'
+  autoload :GraphiteSystemLogger, 'graphiterb/graphite_system_logger'
+end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: graphiterb
 version: !ruby/object:Gem::Version 
+  hash: 27
   prerelease: false
   segments: 
   - 0
-  - 0
   - 1
-  version: 0.0.1
+  - 0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Philip (flip) Kromer (@mrflip)
@@ -14,16 +15,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-02 00:00:00 +00:00
-default_executable: example-client.rb
+date: 2010-07-19 00:00:00 +00:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 13
         segments: 
         - 1
         - 2
@@ -35,9 +38,11 @@ dependencies:
   name: yard
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -46,24 +51,37 @@ dependencies:
 description: Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com
 email: info@infochimps.org
 executables: 
-- example-client.rb
+- run_servers.sh
+- storage_monitor.rb
+- api_call_monitor.rb
+- loadavg_graphite_sender.rb
+- toy.rb
+- file_monitor.rb
 extensions: []
 
 extra_rdoc_files: 
 - LICENSE
-- README.rdoc
+- README.textile
 files: 
 - .document
 - .gitignore
 - CHANGELOG
 - LICENSE
-- README.rdoc
+- README.textile
 - Rakefile
 - VERSION
-- bin/example-client.rb
+- bin/api_call_monitor.rb
+- bin/file_monitor.rb
+- bin/loadavg_graphite_sender.rb
+- bin/run_servers.sh
+- bin/storage_monitor.rb
+- bin/toy.rb
 - graphiterb.gemspec
 - lib/graphiterb.rb
+- lib/graphiterb/graphite_logger.rb
+- lib/graphiterb/graphite_script.rb
 - lib/graphiterb/graphite_sender.rb
+- lib/graphiterb/graphite_system_logger.rb
 - spec/graphiterb_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
@@ -77,26 +95,30 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Fast Ubiquitous Dashboard Logs with Graphite (http://graphite.wikidot.com)
 test_files: 
-- spec/spec_helper.rb
 - spec/graphiterb_spec.rb
+- spec/spec_helper.rb

data/README.rdoc

@@ -1,17 +0,0 @@
-= graphiterb
-
-Description goes here.
-
-== Note on Patches/Pull Requests
- 
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
-
-== Copyright
-
-Copyright (c) 2010 Philip (flip) Kromer (@mrflip). See LICENSE for details.