lpmp 1.0.0

data/lib/exception.rb

@@ -0,0 +1,8 @@
+class AgentNotAlive < RuntimeError;end
+class AgentNotAvailable < RuntimeError;end
+class RequestCodeError < RuntimeError;end
+class CaichenAgentInitError < RuntimeError;end
+class IPNotRegistered < RuntimeError;end
+class ParamsError < RuntimeError;end
+class LoggerNotExist < RuntimeError;end
+class DataExpired < RuntimeError;end

data/lib/lpmp.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+require "pp"
+require File.join(File.dirname(__FILE__),'exception.rb')
+
+begin
+  require 'sinatra/base'
+  require "logger"
+  require "thin"
+  require 'activesupport' unless ({}.respond_to? :to_json)
+rescue LoadError => e
+  p <<DOC
+-----------------------------------------------------------------------
+LPM say:
+The LPM server require sinatra and activesupport.
+Please exec "gem install sinatra" or "gem install activesupport" or "gem install logger" or "gem instasll thin" first.
+-----------------------------------------------------------------------
+Ruby say:
+#{e}
+-----------------------------------------------------------------------
+DOC
+  exit(-1)
+end
+
+module Lpmp
+  def self.run!(vars)
+    raise LoggerNotExist,"logger is needed,$logger in lpmp will pass to vars[:logger].but now vars[:logger] => #{vars[:logger]}" unless vars[:logger]
+    
+    case vars[:mode].to_sym
+    when :PULL
+      vars[:logger].debug "Entry PULL mode"
+      require File.join(File.dirname(__FILE__),'pull','lpmp.rb')
+      LPMPServer.run!('0.0.0.0',vars[:port]||6789)
+    when :BGPULL
+      # not complete.
+      # require File.join(File.dirname(__FILE__),'bg_pull','lpmp.rb')
+    when :XQPUSH
+      vars[:logger].debug "Entry XQPUSH mode"
+      require File.join(File.dirname(__FILE__),'xq_push','lpmp.rb')
+      require File.join(File.dirname(__FILE__),'xq_push','caichen.rb')
+      begin
+        LPMPServer.run(Lpmp::Caichen.new(vars),:port => vars[:port]||6789)
+      rescue ParamsError => e
+        vars[:logger].fatal "XQPUSH mode start failed.Maybe Lpmp::Caichen.new(#{vars.inspect}) error.#{$/}#{e.to_s}."
+        raise e
+      rescue Exception => e
+        p "LPMP start error!"
+        p e.to_s
+      end
+    else
+      vars[:logger].error "No mode fit.vars[:mode] => #{vars[:mode]}.#{$/}Only one of %w[PULL BGPULL XQPUSH] is available."
+    end
+  end
+end

data/lib/pull/lpmp.rb

@@ -0,0 +1,355 @@
+require 'rubygems'
+require "pp"
+require File.join(File.dirname(__FILE__),'..','remote_lmp.rb')
+require File.join(File.dirname(__FILE__),'..','utils.rb')
+
+begin
+  require 'sinatra/base'
+  require "log4r"
+  include Log4r
+  require 'activesupport' unless ({}.respond_to? :to_json)
+rescue LoadError => e
+  p <<DOC
+-----------------------------------------------------------------------
+LPM say:
+The LPM server require sinatra and activesupport.
+Please exec "gem install sinatra" or "gem install activesupport" or "gem install log4r" first.
+-----------------------------------------------------------------------
+Ruby say:
+#{e}
+-----------------------------------------------------------------------
+DOC
+  exit(-1)
+end
+
+$HOSTS = []
+$HostClientObjs = {}
+$Log = nil
+
+def init_config
+  
+=begin rdoc
+  $HostClientObjs => :
+  {'http://10.2.226.20:9876' =>
+    {
+      :proc => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876'>,
+      :process => {
+        "123" => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876' pid=123>,
+        "456" => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876' pid=456>,
+        "789" => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876' pid=789>
+      }
+    }
+  }
+=end
+  
+=begin rdoc
+  $HOSTS => :
+  [{
+    :ip => 'http://10.2.226.20:9876'
+    },{
+    :ip => 'http://10.2.226.37',
+    :pids => %w[123,456,789]
+  }]
+=end
+  File.open( File.join(Lpmp::ROOT,'config','hosts.yaml') ) { |yf| $HOSTS = YAML::load( yf ) }
+  if $HOSTS
+    $HOSTS.each { |host|$HostClientObjs[host[:ip]] = {:proc => RemoteLmp::ProcAgent.new(host[:ip]),:process => (host[:pids]==nil ? [] : host[:pids].inject([]) { |arr, var| arr << RemoteLmp::ProcessAgent.new(host[:ip],var)  })}  }
+  end
+  
+  $Log = Logger.new File.join(Lpmp::ROOT,'log','pull.log')
+  File.open( File.join(Lpmp::ROOT,'config','log.yaml') ) { |yf| $Log.level = YAML::load( yf ) }
+  
+  $HOSTS
+end
+
+def ext_response(bool,msg='OK')
+  
+=begin rdoc
+  {
+      "success":true, // note this is Boolean, not string
+      "msg":"Consignment updated"
+  }
+=end
+  (bool == true) || (bool == 'true') ? "{\"success\":true,\"msg\":\"#{msg}}\"" : "{\"success\":false,\"msg\":\"#{msg}\"}"
+end
+
+def get_hosts_grid_store_data
+  # attributes = [:cpu,:loadavg,:meminfo,:uptime,:df,:date]
+  r = []
+  t = []
+  # p 'get_hosts_grid_store_data\'s HostClientObjs :'
+  # pp $HostClientObjs
+  $HostClientObjs.each { |key,obj|
+    tmp = {}
+    t << Thread.fork{
+      tmp.update({:cpu => obj[:proc].cpu["cpu"]})
+    }
+    tmp.update({
+      :ip => obj[:proc].baseUrl,
+      :hostname => obj[:proc].hostname,
+      :loadavg => obj[:proc].loadavg["load"][0],
+      :mem_total => obj[:proc].meminfo["MemTotal"],#.chomp(' kB'),
+      :mem_free => obj[:proc].meminfo["MemFree"],#.chomp(' kB'),
+      :uptime_total => obj[:proc].uptime["total"],
+      :uptime_idle => obj[:proc].uptime["idle"],
+      :disk_size => obj[:proc].df["size"],
+      :disk_use => obj[:proc].df["use"],
+      :date => obj[:proc].date
+    })
+    # p 'get_hosts_grid_store_data\'s item :'
+    # pp tmp
+    r << tmp
+  }
+  t.each { |e| e.join  }
+  {:root => r,:total => r.size}.to_json
+end
+
+def get_host_process_grid_store_data(ip)
+  # attributes = [:cwd,:exe,:root,:cmdline,:mem]
+  r = []
+  # t = []
+  # p 'get_host_process_grid_store_data\'s HostClientObjs :'
+  # pp $HostClientObjs
+  $HostClientObjs[ip.to_s][:process].each { |key,obj|
+    tmp = {}
+    # t << Thread.fork{
+    #   tmp.update({:cpu => obj.cpu})
+    # }
+    tmp.update({
+      :pid => key, # obj[:process].pid,
+      :cwd => obj.cwd,
+      :exe => obj.exe,
+      :root => obj.root,
+      :cmdline => obj.cmdline,
+      :memory => obj.mem
+    })
+    # p 'get_host_process_grid_store_data\'s item :'
+    # pp tmp
+    r << tmp
+  }
+  # t.each { |e| e.join  }
+  {:root => r,:total => r.size}.to_json
+end
+
+# init config
+init_config
+
+=begin rdoc
+  LPMP is server and provides json data for request from web page.
+=end
+
+class LPMP < Sinatra::Base
+    
+  set :static, true
+  # set :public, File.dirname(__FILE__) + '/public'
+  # set :views, File.dirname(__FILE__) + '/erb'
+  set :public, Lpmp::PUBLIC
+  set :views, Lpmp::ERB
+  
+  get '/' do
+    erb :index
+  end
+  
+  get '/hosts/grid/*' do
+    # => this grid is hosts grid
+    begin
+      get_hosts_grid_store_data
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.inspect)
+    end
+  end
+  
+  get '/hosts/tree/*' do
+    begin
+      # [{text: 'Êý¾Ý±¨±í',children: [{text: '½Ó¿ÚÊýÁ¿±ÈÀýͼ',url: 'StatisticsOfIfaceNumber.jsp',leaf: true},{id:'StatisticsOfTaskId',text: 'ÏîÄ¿²âÊÔÂִζԱÈͼ',url: 'StatisticsOfTask.jsp',leaf: true},{text: '½Ó¿ÚÊýÁ¿ÖÜÆÚ±¨±í',url: 'MonthlyIFaceSummary.jsp',leaf: true}]}]
+      children = $HOSTS.inject([]) { |chd, e|  
+        chd << {:text => e[:ip],:leaf => true,:cls => "file"}
+      }
+      [{:text => "HostsTree",:children => children}].to_json
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  get '/host/process/grid/*' do
+    # => this grid is processes grid
+    begin
+      get_host_process_grid_store_data(params[:ip])
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  # if request is for line chart
+  # {:value=>"value1",:time=>123456}
+  
+  get '/host/proc/cpu/*' do
+    begin
+      if $HostClientObjs[params[:ip]]
+        r = $HostClientObjs[params[:ip]][:proc].send(:cpu,*(params[:splat].split('/')))
+        {:value => r["cpu"].to_f,:time => Time.now.strftime("%m-%d %Hh:%Mm:%Ss")}.to_json
+      else
+        ext_response(false,'request uri format : /host/proc/:method/params1/params2/params3/*?ip=10.2.226.20')
+      end
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  get '/host/proc/loadavg/*' do
+    begin
+      if $HostClientObjs[params[:ip]]
+        r = $HostClientObjs[params[:ip]][:proc].send(:loadavg)
+        {:value => r['load'][0].to_f,:time => Time.now.strftime("%m-%d %Hh:%Mm:%Ss")}.to_json
+      else
+        ext_response(false,'request uri format : /host/proc/:method/params1/params2/params3/*?ip=10.2.226.20')
+      end
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  get '/host/proc/line_chart_store/*' do
+    begin
+      if $HostClientObjs[params[:ip]]
+        tmp = {}
+        r = $HostClientObjs[params[:ip]][:proc].send(:loadavg)
+        tmp[:load] = {:value => r['load'][0].to_f,:time => Time.now.strftime("%m-%d %Hh:%Mm:%Ss")}
+        r = $HostClientObjs[params[:ip]][:proc].send(:cpu,*(params[:splat].split('/')))
+        tmp[:cpu] = {:value => r["cpu"].to_f,:time => Time.now.strftime("%m-%d %Hh:%Mm:%Ss")}
+        tmp.to_json
+      else
+        ext_response(false,'request uri format : /host/proc/:method/params1/params2/params3/*?ip=10.2.226.20')
+      end
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end    
+  end
+  
+  # if request is for pie chart
+  # [{:type=>"str1",:count=>"count1"},
+  # {:type=>"str2",:count=>"count2"},
+  # {:type=>"str3",:count=>"count3"}]
+  
+  get '/host/proc/meminfo/*' do
+    begin
+      if $HostClientObjs[params[:ip]]
+        # p 'mark'
+        r = $HostClientObjs[params[:ip]][:proc].send(:meminfo)
+        # p 'r => :'
+        # pp r
+        t = r.inject([]){|a,e|
+          # pp e
+          a << {:type => e[0],:count => e[1]}
+        }
+        # p "t => :"
+        # pp t
+        "{'root':#{t.to_json},'total':'#{t.size}'}"
+      else
+        ext_response(false,'request uri format : /host/proc/:method/params1/params2/params3/*?ip=10.2.226.20')
+      end
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  get '/host/proc/df/*' do
+    begin
+      if $HostClientObjs[params[:ip]]
+        r = $HostClientObjs[params[:ip]][:proc].send(:df)
+        t = r.inject([]){|a,e|
+          a << {:type => e[0],:count => e[1]}
+        }
+        "{'root':#{t.to_json},'total':'#{t.size}'}"
+      else
+        ext_response(false,'request uri format : /host/proc/:method/params1/params2/params3/*?ip=10.2.226.20')
+      end
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  get '/host/proc/pids/*' do
+    begin
+      if $HostClientObjs[params[:ip]]
+        r = $HostClientObjs[params[:ip]][:proc].send(:pids,*(params[:splat].split('/')))
+        (r.map { |e| {:text => e["pid"],:leaf => true,:cls => "file",:qtip => e["cmdline"]} }).to_json
+      else
+        ext_response(false,'request uri format : /host/proc/:method/params1/params2/params3/*?ip=10.2.226.20')
+      end
+    rescue Exception => e
+      ext_response(false,'sorry,server exception. '+e.to_s+'; $HostClientObjs=>'+$HostClientObjs.to_s)
+    end
+  end
+  
+  get '/register' do
+    ($HOSTS.push params[:host] ? {:ip=>params[:ip],:host=>params[:host]} : {:ip=>params[:ip]}) if params[:ip]
+    begin
+      File.open( File.join(File.dirname(__FILE__),'config','hosts.yaml'), 'w' ) do |out|
+        YAML.dump( $HOSTS, out )
+      end
+      init_config
+      ext_response(true)
+    rescue Exception => e
+      ext_response(false,"can't read/write hosts.yaml")
+    end
+  end
+  
+  get '/reload' do
+    begin
+      ext_response(true,init_config.to_json)
+    rescue Exception => e
+      ext_response(false,"can't read/write hosts.yaml")
+    end
+  end
+  
+  get '/help' do
+    <<-DOC
+    <pre>
+    Welcome using QianNvYouHun(倩女幽魂)
+     -- also named LPMP(Linux Proc Monitor Portal)
+    Any problem you could send mail to zheng.cuizh@gmail.com
+     -- Author:CharlesCui@Alibaba
+     -- MSN:cuizheng.hz@hotmail.com
+    </pre>
+    DOC
+  end
+  
+  get '/test_chart' do
+    <<-DOC
+    <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+       "http://www.w3.org/TR/html4/strict.dtd">
+
+    <html lang="en">
+    <head>
+    	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    	<title>Linux Proc Monitor Portal</title>
+    	<meta name="generator" content="TextMate http://macromates.com/">
+    	<meta name="author" content="Charles">
+    	<link rel="shortcut icon" href="http://www.alisoft.com/cms/apps/img/favicon.ico" type="image/x-icon" /> 
+    	<link href="/ext-3.0.0/resources/css/ext-all.css"  media="screen" rel="stylesheet" type="text/css" />
+    	<script type="text/javascript" src="/ext-3.0.0/adapter/ext/ext-base.js" charset="utf-8"></script>
+    	<script type="text/javascript" src="/ext-3.0.0/ext-all-debug.js" charset="utf-8"></script>
+    	<script type="text/javascript" src="/ext-3.0.0/source/locale/ext-lang-zh_CN.js" charset="utf-8"></script>
+    	
+    	<script src="/js/global.js" type="text/javascript" charset="utf-8"></script>
+    	<script src="/js/test_chart.js" type="text/javascript" charset="utf-8"></script>
+    </head>
+    一定要有个容器!
+    <br/>
+    <div id="container"/>
+    <br/>
+    还要有毅力!
+    <br/>
+    <div id ="disk"/>
+    </html>
+    DOC
+  end
+end
+
+
+class LPMPServer
+  def self.run(host='0.0.0.0',port=16789)
+    Daemon.start{LPMP.run! :host => host, :port => port}
+  end
+end

data/lib/remote_lmp.rb

@@ -0,0 +1,129 @@
+require 'rubygems'
+require File.join(File.dirname(__FILE__),'exception.rb')
+
+begin
+  gem 'sinatra'
+  require 'sinatra/base'
+  require 'activesupport'
+  require 'httparty'
+rescue LoadError => e
+  p <<DOC
+-----------------------------------------------------------------------
+LPM say:
+The LPM server require sinatra and activesupport.
+Please exec "gem install sinatra" or "gem install activesupport" first.
+"gem install httparty" is needed as well
+-----------------------------------------------------------------------
+Ruby say:
+#{e}
+-----------------------------------------------------------------------
+DOC
+  exit(-1)
+end
+
+module RemoteLmp
+  
+  class ProcAgent
+    include HTTParty
+    attr :baseUrl
+  
+    def initialize(url)
+      # @baseUri = URI.parse url
+    
+      # => @baseUrl = 'http://10.2.226.20:9876'
+      # >> @baseUrl[-1..-1]
+      # => "/"
+      # >> @baseUrl[-1]
+      # => 47
+      @baseUrl = (url[-1] == 47 ? url[0..-2] : url)
+      @baseUrl = (@baseUrl[0..6]=='http://' ? @baseUrl : "http://#{@baseUrl}")
+      raise AgentNotAvailable,"Agent is running but with some error,maybe your url is error." unless heart_beat
+    end
+ 
+    # method_missing instead of the common request,
+    # such as: cpu,loadavg...
+    # def cpu(*arguments)
+    #   self.get(@baseUrl+'/proc/cpu/'+arguments.join('/'))
+    # end
+    # 
+    # def loadavg(*arguments)
+    #   self.get(@baseUrl+'/proc/loadavg/'+arguments.join('/'))
+    # end
+  
+    def method_missing(method_name,*arguments)
+      r = self.class.get("#{@baseUrl}/proc/#{method_name}/#{arguments.join('/')}")
+      # pp r
+      begin
+        raise RequestCodeError,"this request code is #{r.code.to_s}" unless r.code.to_s =~ /[23]\d\d/
+        ActiveSupport::JSON.decode r
+      # rescue ActiveSupport::JSON::ParseError => e
+      #   r
+      rescue RequestCodeError => e
+        p e+';RequestCodeError => '+r.code
+        nil
+      rescue Exception => e
+        p e
+        r
+      end
+    end
+
+    # below methods is private
+    def heart_beat
+      begin
+        HTTParty.get(@baseUrl).code.to_s =~ /[23]\d\d/ ? true : false
+      rescue Errno::ECONNREFUSED => e
+        raise AgentNotAlive,"Agent maybe down,plz check `curl -v #{@baseUrl}/`" 
+      end
+    end
+    
+    private :heart_beat
+  end
+
+  class ProcessAgent
+    include HTTParty
+    attr :baseUrl
+    attr :pid
+  
+    def initialize(url,pid)
+      @baseUrl = (url[-1] == 47 ? url[0..-2] : url)
+      @baseUrl = (@baseUrl[0..6]=='http://' ? @baseUrl : "http://#{@baseUrl}")
+      @pid = pid
+      raise AgentNotAvailable,"Agent is running but with some error,maybe your url is error." unless heart_beat
+    end
+ 
+    def method_missing(method_name,*arguments)
+      r = self.class.get("#{@baseUrl}/process/#{@pid}/#{method_name}/#{arguments.join('/')}").gsub(/\000/,' ')
+      raise RequestCodeError,"this request cod is #{r.code.to_s}" unless r.code.to_s =~ /[23]\d\d/
+      
+      begin
+        raise RequestCodeError,"this request cod is #{r.code.to_s}" unless r.code.to_s =~ /[23]\d\d/
+        ActiveSupport::JSON.decode r
+      # rescue ActiveSupport::JSON::ParseError => e
+      #   r
+      rescue RequestCodeError => e
+        p e+';RequestCodeError => '+r.code
+        nil
+      rescue Exception => e
+        p e
+        r
+      end
+    end
+
+    # below methods is private
+    def heart_beat
+      begin
+        HTTParty.get(@baseUrl).code.to_s =~ /[23]\d\d/ ? true : false
+      rescue Errno::ECONNREFUSED => e
+        raise AgentNotAlive,"Agent maybe down,plz check `curl -v #{@baseUrl}/`" 
+      end
+    end
+
+    private :heart_beat
+  end
+
+end
+
+
+
+
+