graphite-api 0.0.0.21

data/README.md

@@ -0,0 +1,80 @@
+# GraphiteAPI
+
+A Ruby API toolkit for [Graphite](http://graphite.wikidot.com/)
+
+* Graphite ruby client.
+* Graphite middleware server, a lightweight, event-driven, aggregator daemon.
+
+## Why using it?
+Written in ruby
+Simple and light-width (only one gem dependency)
+Comes with built-in middleware server
+
+## Client Usage
+```ruby
+ 	require 'graphite-api'
+
+ 	client = GraphiteAPI::Client.new(
+	 :host => "graphite.example.com",
+ 	 :port => 2003,
+ 	 :prefix => ["example","prefix"], # add example.prefix to each key
+ 	 :interval => 60                  # send to graphite every 60 seconds
+ 	)
+    
+ 	# Simple:
+ 	client.add_metrics("webServer.web01.loadAvg" => 10.7)
+ 	# => example.prefix.webServer.web01.loadAvg 10.7 time.now.stamp
+	
+ 	# Multiple with time:
+ 	client.add_metrics({
+ 		"webServer.web01.loadAvg" => 10.7,
+ 		"webServer.web01.memUsage" => 40
+ 	},Time.at(1326067060))
+ 	# => example.prefix.webServer.web01.loadAvg  10.7 1326067060
+ 	# => example.prefix.webServer.web01.memUsage 40 1326067060
+ 	
+ 	# Every 10 sec
+ 	client.every(10) do
+ 	  client.add_metrics("webServer.web01.uptime" => `uptime`.split.first.to_i) 
+ 	end
+	
+ 	client.join # wait...
+```	
+## Middleware Usage
+
+`graphite-middleware --help`
+
+```
+Graphite Middleware Server
+
+Usage: graphite-middleware [options]
+    -g, --graphite HOST              graphite host
+    -p, --port PORT                  listening port (default 2003)
+    -l, --log-file FILE              log file
+    -L, --log-level LEVEL            log level (default warn)
+    -P, --pid-file FILE              pid file (default /var/run/graphite-middleware.pid)
+    -d, --daemonize                  run in background
+    -i, --interval INT               report every X seconds (default 60)
+    -s, --slice SECONDS              send to graphite in X seconds slices (default is 60)
+    -c, --cache HOURS                cache expiration time in hours (default is 12 hours)
+```
+
+## Installation
+install the latest from github
+
+```
+git clone git://github.com/kontera-technologies/graphite-api.git
+cd graphite-api
+rake install
+```
+
+## Bugs
+
+If you find a bug, feel free to report it @ our [issues tracker](https://github.com/kontera-technologies/graphite-api/issues) on github.
+
+## License
+
+It is free software, and may be redistributed under the terms specified in [LICENSE](https://github.com/kontera-technologies/graphite-api/blob/master/LICENSE).
+
+## Warranty
+This software is provided “as is” and without any express or implied warranties, including, without limitation, the implied warranties of merchantability and fitness for a particular purpose.

data/Rakefile

@@ -0,0 +1,11 @@
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+Dir.chdir File.dirname __FILE__
+
+require 'graphite-api'
+require 'bundler/setup'
+
+def msg m
+  $stderr.puts "[*] #{m}"
+end
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/bin/graphite-middleware

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..','lib')
+require 'graphite-api'
+
+GraphiteAPI::Runner.new(ARGV).run

data/lib/graphite-api.rb

@@ -0,0 +1,18 @@
+module GraphiteAPI
+  ROOT = File.expand_path(File.dirname(__FILE__))
+
+  autoload :Version,    "#{ROOT}/graphite-api/version"
+  autoload :Client,     "#{ROOT}/graphite-api/client"
+  autoload :Scheduler,  "#{ROOT}/graphite-api/scheduler"
+  autoload :Connector,  "#{ROOT}/graphite-api/connector"
+  autoload :Middleware, "#{ROOT}/graphite-api/middleware"
+  autoload :Runner,     "#{ROOT}/graphite-api/runner"
+  autoload :Utils,      "#{ROOT}/graphite-api/utils"
+  autoload :Buffer,     "#{ROOT}/graphite-api/buffer"
+  autoload :Logger,     "#{ROOT}/graphite-api/logger"
+    
+  def self.version
+    GraphiteAPI::Version.string
+  end
+  
+end

data/lib/graphite-api/buffer.rb

@@ -0,0 +1,110 @@
+# -----------------------------------------------------
+# Buffer Object
+# Handle Socket & Client data streams
+# -----------------------------------------------------
+# Usage:
+#     buff = GraphiteAPI::Buffer.new(GraphiteAPI::Utils.default_options)
+#     buff << {:metric => {"load_avg" => 10},:time => Time.now}
+#     buff << {:metric => {"load_avg" => 30},:time => Time.now}
+#     buff.stream "mem.usage 1"
+#     buff.stream "90 1326842563\n"
+#     buff.stream "shuki.tuki 999 1326842563\n"
+#     buff.each {|o| p o} 
+#
+# Produce:
+#    ["load_avg", 40.0, 1326881160]
+#    ["mem.usage", 190.0, 1326842520]
+#    ["shuki.tuki", 999.0, 1326842520]
+# -----------------------------------------------------
+module GraphiteAPI
+  class Buffer
+    attr_reader :leftovers,:options,:new_records,:in_cache_mode
+
+    def initialize(options)
+      @options = options
+      @leftovers = Hash.new {|h,k| h[k] = Array.new}
+      @new_records = []
+      @in_cache_mode = !options[:cache_exp].nil?
+      start_cleaner if in_cache_mode
+    end
+
+    def << hash
+      time = Utils.normalize_time(hash[:time],options[:slice])
+      hash[:metric].each do |k,v|
+        buffer[time][k] += v.to_f
+        new_records << [time,k]
+      end
+    end
+
+    def stream(data,client_id = nil)
+      got_leftovers = data[-1,1] != "\n"
+      data = data.split(/\n/)
+      unless leftovers[client_id].empty?
+        if (valid leftovers[client_id].last + data.first rescue nil)
+          data.unshift(leftovers[client_id].pop + data.shift)
+        end
+        leftovers[client_id].clear
+      end
+
+      leftovers[client_id] << data.pop if got_leftovers
+      data.each do |line|
+        next unless valid line
+        key,val,time = line.split
+        self << {:metric => {key => val},:time => (Time.at(time.to_i) rescue Time.now)}
+      end
+    end
+
+    def each
+      new_records.uniq.each do |time,key|
+        yield [prefix + key,buffer[time][key],time]
+      end and clear
+    end
+    
+    def empty?
+      buffer.empty?
+    end
+
+    def got_new_records?
+      !new_records.empty?
+    end
+
+    def size
+      buffer.values.map(&:values).flatten.size
+    end
+
+    private
+    def clear
+      new_records.clear
+      buffer.clear unless in_cache_mode
+    end
+    
+    def valid(data)
+      data =~ /^[\w|\.]+ \d+(?:\.|\d)* \d+$/
+    end
+    
+    def prefix
+      @prefix ||= options[:prefix].empty? ? String.new : prefix_to_s 
+    end
+    
+    def prefix_to_s
+      [options[:prefix]].flatten.join('.') << "."
+    end
+    
+    def buffer
+      @buffer ||= Hash.new {|h,k| h[k] = Hash.new {|h1,k1| h1[k1] = 0}}
+    end
+
+    def clean(age)
+      [buffer,new_records].each {|o| o.delete_if {|t,k| now - t > age}}
+    end
+    
+    def now
+      Time.now.to_i
+    end
+    
+    def start_cleaner
+      Scheduler.every(options[:cleaner_interval]) { clean options[:cache_exp] }
+    end
+
+  end
+end

data/lib/graphite-api/client.rb

@@ -0,0 +1,70 @@
+# -----------------------------------------------------
+# Graphite Client
+# Send metrics to graphite (or to some kind of middleware/proxy) 
+# -----------------------------------------------------
+# Usage
+#
+#  client = GraphiteAPI::Client.new(
+#    :host => "graphite.example.com",
+#    :port => 2003,
+#    :prefix => ["example","prefix"], # add example.prefix to each key
+#    :interval => 60                  # send to graphite every 60 seconds
+#    )
+#  
+#  # Simple:
+#  client.add_metrics("webServer.web01.loadAvg" => 10.7)
+#  # => example.prefix.webServer.web01.loadAvg 10.7 time.now.stamp
+#  
+#  # Multiple with time:
+#  client.add_metrics({
+#	  "webServer.web01.loadAvg" => 10.7,
+#	  "webServer.web01.memUsage" => 40
+#  },Time.at(1326067060))
+#  # => example.prefix.webServer.web01.loadAvg  10.7 1326067060
+#  # => example.prefix.webServer.web01.memUsage 40 1326067060
+#  
+#  # Every 10 sec
+#  client.every(10) do
+#    client.add_metrics("webServer.web01.uptime" => `uptime`.split.first.to_i) 
+#  end
+#  
+#  client.join # wait...
+# -----------------------------------------------------
+module GraphiteAPI
+  class Client
+    attr_reader :options,:buffer,:connector
+    
+    def initialize(opt)
+      @options   = GraphiteAPI::Utils.default_options.merge opt
+      @buffer    = GraphiteAPI::Buffer.new(options)
+      @connector = GraphiteAPI::Connector.new(*options.values_at(:host,:port))
+      start_scheduler
+    end
+
+    def add_metrics(m,time = Time.now)
+      buffer << {:metric => m, :time => time}
+    end
+
+    def join
+      sleep 1 while buffer.got_new_records?
+    end
+    
+    def stop
+      Scheduler.stop
+    end
+    
+    def every(frequency,&block)
+      Scheduler.every(frequency,&block)
+    end
+
+    protected
+    def start_scheduler
+      Scheduler.every(options[:interval]) {send_metrics}
+    end
+    
+    def send_metrics
+      buffer.each {|arr| connector.puts arr.join(" ")} 
+    end 
+    
+  end
+end

data/lib/graphite-api/connector.rb

@@ -0,0 +1,41 @@
+# -----------------------------------------------------
+# TCP Socket connection
+# -----------------------------------------------------
+# Usage:
+#    connector = GraphiteAPI::Connector.new("localhost",2003)
+#    connector.puts("my.metric 1092 1232123231")
+#
+# Socket:
+# => my.metric 1092 1232123231\n
+# -----------------------------------------------------
+require 'socket'
+module GraphiteAPI
+  class Connector
+    attr_reader :options
+    
+    def initialize(host,port)
+      @host = host
+      @port = port
+    end
+    
+    def puts(msg)
+      begin
+        Logger.debug msg
+        socket.puts(msg)
+      rescue Errno::EPIPE
+        @socket = nil
+      retry
+      end
+    end
+    
+    protected
+    
+    def socket
+      if @socket.nil? || @socket.closed?
+        @socket = TCPSocket.new(@host,@port)
+      end
+      @socket
+    end
+     
+  end
+end

data/lib/graphite-api/logger.rb

@@ -0,0 +1,18 @@
+# -----------------------------------------------------
+# GraphiteAPI Logger
+# -----------------------------------------------------
+# Usage:
+#     Graphite::Logger.logger = ::Logger.new(STDOUT) 
+#     Graphite::Logger.info "shuki tuki"
+#     Graphite::Logger.debug "hihi"
+# -----------------------------------------------------
+module GraphiteAPI
+  class Logger
+    class << self
+      attr_accessor :logger
+      def method_missing(m,*args,&block)
+        if logger.respond_to? m then logger.send(m,*args,&block) end
+      end      
+    end
+  end
+end

data/lib/graphite-api/middleware.rb

@@ -0,0 +1,74 @@
+# -----------------------------------------------------
+# Graphite Middleware Server
+# Should be placed between graphite server and graphite clients
+# - Data Aggregator
+# - Caching
+# - Data Manipulation
+# -----------------------------------------------------
+# Usage:
+#   GraphiteAPI::Middleware.start(options)
+# 
+# Options:
+#   graphite_host    target graphite hostname
+#   graphite_port    target graphite listening port (default 2003)
+#   cache_exp        cache lifetime in seconds  (default is 43200 seconds)
+#   prefix           add prefix to each key
+#   interval         report to graphite every X seconds (default is 60)
+#   slice            send to graphite in X seconds slices (default is 60)
+#   log_level        info # TODO: move this to runner.rb
+# -----------------------------------------------------
+require 'rubygems'
+require 'eventmachine'
+require 'logger'
+require 'socket'
+
+module GraphiteAPI
+  class Middleware < EventMachine::Connection
+
+    attr_reader :logger,:buffer,:client_id
+
+    def initialize(logger,buffer)
+      @logger = logger
+      @buffer = buffer
+      super
+    end
+
+    def post_init
+      logger.debug "Client connecting"
+      @client_id = Socket.unpack_sockaddr_in(get_peername).join(":")
+    end
+
+    def receive_data(data)
+      buffer.stream(data,client_id)
+    end
+
+    def unbind
+      logger.debug "Client disconnecting"
+    end
+
+    def self.start(options)
+      EventMachine.run do
+        # Resources
+        GraphiteAPI::Logger.logger = ::Logger.new(options[:log_file] || STDOUT)
+        GraphiteAPI::Logger.level = eval "::Logger::#{options[:log_level].to_s.upcase}"
+        
+        buffer    = GraphiteAPI::Buffer.new(options)
+        connector = GraphiteAPI::Connector.new(*options.values_at(:graphite_host,:graphite_port))
+        
+        # Starting server
+        EventMachine.start_server('0.0.0.0',options[:listening_port],self,GraphiteAPI::Logger.logger,buffer)
+        GraphiteAPI::Logger.info "Server running on port #{options[:listening_port]}"
+        
+        # Send metrics to graphite every X seconds
+        GraphiteAPI::Scheduler.every(options[:interval]) do
+          if buffer.got_new_records?
+            Logger.debug "Sending #{buffer.size} records to graphite (@#{options[:graphite_host]}:#{options[:graphite_port]})"
+            buffer.each { |arr| connector.puts arr.join(" ") }
+          end # if got_new_records?
+        end # every 
+      end # run 
+    end # start
+  end # Middleware
+end # GraphiteAPI
+#require File.dirname(__FILE__) + "/../graphite-api"
+#GraphiteAPI::Middleware.start(GraphiteAPI::Utils.default_options)

data/lib/graphite-api/runner.rb

@@ -0,0 +1,69 @@
+require 'optparse'
+
+module GraphiteAPI
+  class Runner
+    attr_reader :options
+
+    def initialize(argv)
+      parser.parse! argv
+      validate_options
+    end
+    
+    def run
+      if options[:daemonize]
+        fork do
+          Process.setsid
+          exit if fork
+          Dir.chdir('/tmp')
+          STDOUT.reopen('/dev/null','a')
+          STDIN.reopen('/dev/null')
+          STDERR.reopen('/dev/null','a')
+          write_pid
+          run!
+        end
+      else
+        run!
+      end
+    end
+
+    private
+    def options
+      @options ||= Utils.default_options
+    end
+    
+    def write_pid
+      begin
+        File.open(options[:pid], 'w') { |f| f.write(Process.pid) }
+      rescue Exception
+      end
+    end
+
+    def run!
+      GraphiteAPI::Middleware.start(options)
+    end
+
+    def validate_options
+      abort "You must specify graphite host" if options[:graphite_host].nil?
+    end
+
+    def parser
+      OptionParser.new do |opts| 
+        opts.banner = "Graphite Middleware Server"
+        opts.define_head "Usage: graphite-middleware [options]"
+        opts.define_head ""
+        opts.on("-g", "--graphite HOST","graphite host") {|v| options[:graphite_host] = v}        
+        opts.on("-p", "--port PORT","listening port (default 2003)"){|v| options[:listening_port] = v}
+        opts.on("-l", "--log-file FILE","log file") {|v| options[:log_file] = File.expand_path(v)}
+        opts.on("-L", "--log-level LEVEL","log level (default warn)") {|v|options[:log_level] = v}
+        opts.on("-P", "--pid-file FILE","pid file (default /var/run/graphite-middleware.pid)"){|v|options[:pid] = v}
+        opts.on("-d", "--daemonize","run in background"){options[:daemonize] = true}
+        opts.on("-i", "--interval INT","report every X seconds (default 60)"){|v|options[:interval] = v.to_i unless v.to_i == 0}
+        opts.on("-s", "--slice SECONDS","send to graphite in X seconds slices (default is 60)") {|v| options[:slice] = v.to_i unless v.to_i == 0}
+        opts.on("-c", "--cache HOURS","cache expiration time in hours (default is 12 hours)") {|v| (options[:cache_exp] = v.to_i * 3600) unless v.to_i == 0}
+        opts.define_tail ""
+        opts.define_tail ""
+      end
+    end
+
+  end
+end

data/lib/graphite-api/scheduler.rb

@@ -0,0 +1,42 @@
+require 'rubygems'
+require 'eventmachine'
+
+module GraphiteAPI
+  class Scheduler
+    @@wrapper = nil
+    @@timers  = []
+    
+    class << self
+      def every(frequency,&block)
+        reactor_running? or start_reactor
+        timers << EventMachine::PeriodicTimer.new(frequency) { yield }
+        timers.last
+      end
+
+      def stop
+        timers.map(&:cancel)
+        wrapper and EventMachine.stop
+      end
+
+      def join
+        wrapper and wrapper.join
+      end
+
+      private
+      def start_reactor
+        @@wrapper = Thread.new { EventMachine.run }
+        wrapper.abort_on_exception = true
+        Thread.pass until reactor_running?
+      end
+
+      def reactor_running?
+        EventMachine.reactor_running?
+      end
+      
+      def wrapper;@@wrapper end
+      def timers; @@timers end
+      
+    end
+    
+  end
+end

data/lib/graphite-api/utils.rb

@@ -0,0 +1,26 @@
+module GraphiteAPI
+  module Utils
+    module_function
+
+    def normalize_time(time,slice = 60)
+      ((time || Time.now).to_i / slice * slice).to_i
+    end
+    
+    def default_options
+      {
+        :cleaner_interval => 43200,
+        :graphite_port => 2003,
+        :listening_port => 2003,
+        :log_level => :info,
+        :cache_exp => nil,
+        :host => "localhost",
+        :port => 2003,
+        :prefix => [],
+        :interval => 60,
+        :slice => 60,
+        :pid => "/var/run/graphite-middleware.pid"
+      }
+    end
+    
+  end
+end

data/lib/graphite-api/version.rb

@@ -0,0 +1,22 @@
+module GraphiteAPI
+  class Version
+    PFILE = File.join(GraphiteAPI::ROOT,"..",".pre_version")
+    class << self
+      def string
+        [0,0,0,pre].join(".")
+      end
+
+      def increment_pre
+        new_version = pre + 1
+        File.open(PFILE,"w") { |f| f.puts new_version }
+      end
+
+      private
+      def pre
+        File.read(PFILE).to_i
+      end
+      
+    end
+    
+  end
+end

data/tasks/build.rake

@@ -0,0 +1,49 @@
+require 'rubygems/package_task'
+
+GraphiteAPI::GemSpec = Gem::Specification.new do |s|
+  s.name                  = "graphite-api"
+  s.version               = GraphiteAPI.version
+  s.platform              = Gem::Platform::RUBY
+  s.summary               = "GraphiteAPI"
+  s.description           = "GraphiteAPI - Graphite API tools"
+  s.author                = "Eran Levi"
+  s.email                 = 'eran@kontera.com'
+  s.homepage              = 'http://www.kontera.com'
+  s.executables           = %w(graphite-middleware)
+  s.required_ruby_version = '>= 1.8.7'
+  s.rubyforge_project     = "graphite-api"
+  s.files                 = %w(README.md Rakefile) + Dir.glob("{bin,lib,test,tasks}/**/*")
+  s.require_path          = "lib"
+  s.bindir                = "bin"
+
+  s.add_dependency 'eventmachine','>= 0.3.3'
+end
+
+task :gem => [:clobber_package]
+Gem::PackageTask.new(GraphiteAPI::GemSpec) do |p|
+  p.gem_spec = GraphiteAPI::GemSpec
+end
+
+task :install => [:gem] do
+   sh "gem install pkg/graphite-api"
+   Rake::Task['clobber_package'].execute
+end
+
+namespace :gem do
+  desc "Update GraphiteAPI gem version"
+  task :update_version do
+    GraphiteAPI::Version.increment_pre
+    msg "Incrementing version to #{GraphiteAPI::Version.string}..."
+  end
+  
+  desc "Upload GraphiteAPI to Kontera's repo"
+  task :upload => [:build] do
+    sh "gem inabox --host http://gems.kontera.com pkg/graphite-api-#{GraphiteAPI.version}.gem"
+  end
+  
+  desc "Update GraphiteAPI gem version and build gem"
+  task :build => [:test,:update_version] do
+    sh "rake gem"
+  end 
+  
+end

data/tasks/tests.rake

@@ -0,0 +1,3 @@
+task :test do
+  # tests here
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: graphite-api
+version: !ruby/object:Gem::Version
+  version: 0.0.0.21
+  prerelease: 
+platform: ruby
+authors:
+- Eran Levi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-27 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &2152684380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152684380
+description: GraphiteAPI - Graphite API tools
+email: eran@kontera.com
+executables:
+- graphite-middleware
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- bin/graphite-middleware
+- lib/graphite-api/buffer.rb
+- lib/graphite-api/client.rb
+- lib/graphite-api/connector.rb
+- lib/graphite-api/logger.rb
+- lib/graphite-api/middleware.rb
+- lib/graphite-api/runner.rb
+- lib/graphite-api/scheduler.rb
+- lib/graphite-api/utils.rb
+- lib/graphite-api/version.rb
+- lib/graphite-api.rb
+- tasks/build.rake
+- tasks/tests.rake
+homepage: http://www.kontera.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: graphite-api
+rubygems_version: 1.8.13
+signing_key: 
+specification_version: 3
+summary: GraphiteAPI
+test_files: []