lpmp 1.0.0

data/LICENSE

@@ -0,0 +1,3 @@
+== lpmp
+
+Put appropriate LICENSE for your project here.

data/README

@@ -0,0 +1,3 @@
+== lpmp
+
+You should document your project here.

data/Rakefile

@@ -0,0 +1,45 @@
+# 
+# To change this template, choose Tools | Templates
+# and open the template in the editor.
+ 
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+
+spec = Gem::Specification.new do |s|
+  s.name = 'lpmp'
+  s.version = '1.0.0'
+  s.has_rdoc = true
+  s.extra_rdoc_files = ['README', 'LICENSE']
+  s.summary = 'Linux Proc Monitor Portal'
+  s.description = s.summary
+  s.author = 'Charles Cui'
+  s.email = 'zheng.cuizh@gmail.com'
+  s.executables = ['lpmp']
+  s.files = %w(LICENSE README Rakefile) + Dir.glob("{bin,config,lib,spec,erb,log,public,test}/**/*")
+  s.require_path = "lib"
+  s.bindir = "bin"
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.gem_spec = spec
+  p.need_tar = true
+  p.need_zip = true
+end
+
+Rake::RDocTask.new do |rdoc|
+  files =['README', 'LICENSE', 'lib/**/*.rb']
+  rdoc.rdoc_files.add(files)
+  rdoc.main = "README" # page to start on
+  rdoc.title = "lpmp Docs"
+  rdoc.rdoc_dir = 'doc/rdoc' # rdoc output folder
+  rdoc.options << '--line-numbers'
+end
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/**/*.rb']
+end

data/bin/lpmp

@@ -0,0 +1,107 @@
+# lpmp is same as caichen
+
+require 'rubygems'
+require 'optparse'
+# require "log4r"
+require "logger"
+
+require File.join(File.dirname(__FILE__),'..','lib','lpmp.rb')
+
+module Lpmp
+  ROOT = File.join(File.dirname(__FILE__),'..')
+  ERB = File.join(File.dirname(__FILE__),'..','erb')
+  PUBLIC = File.join(File.dirname(__FILE__),'..','public')
+end
+
+
+vars = {}
+ARGV.options do
+  |opts|
+  opts.banner = <<DOC
+  CaiChen [also named => L(inux) P(roc) M(onitor) P(ortal)] :
+  Mail/Gtalk: zheng.cuizh@gmail.com
+  
+  Usage: #{File.basename($0)} [options] argv...
+  (sudo )lpmp --memcache=10.2.226.133:11011,10.2.226.133:11012 --log_level=0 --mode=XQPUSH
+  
+  Attention:sync LPM server and LPMP server datetime. command : ntpdate 10.2.228.8
+DOC
+
+  opts.on_head("LPMP optins useage:")
+
+  vars[:port] = 6789
+  opts.on("-p", "--port=PORT", Integer,
+	  "lpmp server requires a port number.","default port=#{vars[:port]}") {|vars[:port]|}
+	
+  # separater
+	opts.on()
+  vars[:mode] = 'PULL'
+  opts.on("--mode [MODE]",[:PULL,:BGPULL,:XQPUSH],
+	  "PULL : (default) lpmp server will request data from lpm server when a request coming.",
+	  "       This mode require a high performance lpmp server,it's used to be a daemon.",
+	  "BGPULL : This mode will run some background process at lpmp server.",
+	  "         It needs memcaches.see --memcache=[LIST,LIST]",
+	  "XQPUSH : This mode will get data from memcache when a request come from client.",
+	  "         This mode is the heighest performance of the three.",
+	  "         It also needs memcaches.see --memcache=[LIST,LIST]",
+	  "default --mode=#{vars[:mode]}.") {|vars[:mode]|}
+	  
+  # array
+  opts.on("-m", "--memcache=LIST,LIST", Array,
+    "memcache list,such as : --memcache=10.2.226.38:11211,10.2.226.37:11211","memcache server(s) must be alive.") {|vars[:memcache]|}
+  
+  vars[:log] = "/var/log/lpmp.log"
+  opts.on("-l","--log=LogPath",
+    "default --log=#{vars[:log]}"){|vars[:log]|}
+    
+  vars[:default_log_level] = 3
+  opts.on("--log_level=LogLevel",Integer,"1 => DEBUG","2 => INFO","3 => WARN","4 => ERROR","5 => FATAL",
+    "default --log_level=#{vars[:default_log_level]}"){|vars[:log_level]|}
+
+  vars[:timeout] = 60
+  opts.on("-t=60",Integer,"Make sure the LPMP server's datatime is same as LPM server,",
+    "or the difference is little than timeout,else the memcache data will not available for LPMP",
+    "(sudo )ntpdate 10.2.228.8(your date server ip)",
+    "default -t=#{vars[:timeout]}"){|vars[:timeout]|}
+    
+  # separater
+  opts.on_tail
+  opts.on_tail("common options:")
+
+  # no argument, shows at tail
+  opts.on_tail("--help", "show this message") {puts opts; exit}
+
+  # no argument
+  opts.on_tail("--version", "show version") do
+    puts "CaiChen [L(inux) P(roc) M(onitor) P(ortal)] : 1.0.1"
+    exit
+  end
+  opts.parse!
+end
+
+vars[:logger] = nil
+
+begin
+  # vars[:logger] = Logger.new(vars[:log])
+  Logger::Formatter::Format = "%s, [%s#%d] %5s -- %s: %s#{$/}"
+  vars[:logger] = Logger.new(vars[:log],'daily',1048576)
+  if vars[:log_level] && (vars[:log_level] >= 1 || vars[:log_level] <= 5) then
+    vars[:logger].level = vars[:log_level]
+  else
+    vars[:logger].level = vars[:default_log_level]
+  end
+  vars[:logger].info "Logger is created.#{vars[:logger]}"
+rescue Exception => e
+  vars[:logger] = nil
+  p "params error:#{e.to_s}"
+  p "can't access log file => #{vars[:log]}"
+end
+
+vars[:logger].info "LPMPServer is starting..."
+vars[:logger].info "Server parameters are below : #{vars.inspect}"
+vars[:logger].info "LPMPServer will running @ 0.0.0.0:#{vars[:port]}"
+
+if vars[:mode] then
+  vars[:logger].info "LPMPServer will run with #{vars[:mode]} mode."
+  Lpmp.run!(vars)
+end

data/config/help.yaml

@@ -0,0 +1 @@
+--- "    Welcome using QianNvYouHun()\n     -- also named LPMP(Linux Proc Monitor Portal)\n    Any problem you could send mail to zheng.cuizh@gmail.com\n     -- Author:CharlesCui@Alibaba\n     -- MSN:cuizheng.hz@hotmail.com\n"

data/config/hosts.yaml

@@ -0,0 +1,2 @@
+--- 
+- :ip: http://10.2.226.133:9876

data/config/log.yaml

@@ -0,0 +1 @@
+--- 4

data/config/memcache.yaml

@@ -0,0 +1,5 @@
+--- 
+- http://10.2.226.20:10001
+- http://10.2.226.20:10002
+- http://10.2.226.20:10003
+- http://10.2.226.20:10004

data/erb/index.erb

@@ -0,0 +1,62 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+   "http://www.w3.org/TR/html4/strict.dtd">
+
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<title>Linux Proc Monitor Portal</title>
+	<meta name="generator" content="TextMate http://macromates.com/">
+	<meta name="author" content="Charles">
+	<link rel="shortcut icon" href="http://www.alisoft.com/cms/apps/img/favicon.ico" type="image/x-icon" /> 
+	<link href="/ext-3.0.0/resources/css/ext-all.css"  media="screen" rel="stylesheet" type="text/css" />
+	<script type="text/javascript" src="/ext-3.0.0/adapter/ext/ext-base.js" charset="utf-8"></script>
+	<script type="text/javascript" src="/ext-3.0.0/ext-all-debug.js" charset="utf-8"></script>
+	<script type="text/javascript" src="/ext-3.0.0/source/locale/ext-lang-zh_CN.js" charset="utf-8"></script>
+	
+	<!-- Below is for portal -->
+	<link rel="stylesheet" type="text/css" href="/css/ext-portal.css" />
+	<script type="text/javascript" src="/js/ext-portal.js"></script>
+	
+	<!-- Self css file -->
+	<link rel="stylesheet" type="text/css" href="/css/lpmp.css" />
+	
+	<!-- Self js file -->
+	<script src="/js/global.js" type="text/javascript" charset="utf-8"></script>
+	<script src="/js/outerframe.js" type="text/javascript" charset="utf-8"></script>
+	<script src="/js/portal.js" type="text/javascript" charset="utf-8"></script>
+	<!-- Date: 2009-09-19 -->
+	<script type="text/javascript" charset="utf-8">
+		<!--
+		Ext.onReady(function(){
+			/*
+			*enable Ext.Fx
+			**/
+			Ext.enableFx = true;
+			
+			/*
+			*self defined js file init()
+			*/
+			outer_frame();
+			portal_init();
+			
+			Ext.state.Manager.setProvider(new Ext.state.CookieProvider());
+			
+			Ext.QuickTips.init();
+			new Ext.Viewport({
+				enableTabScroll:true,
+			    layout: 'border',
+			    items: [
+			      Glb.outer_west,
+			      Glb.outer_north,
+			      Glb.outer_center,
+			      Glb.outer_south
+			    ]
+			});
+			// Glb.inner_north.body.update('');
+		})
+		//-->
+	</script>
+</head>
+<body>
+</body>
+</html>

data/lib/bg_pull/caichen.rb

@@ -0,0 +1,199 @@
+class Caichen
+  attr :cach
+  attr :HOSTS
+  attr :HostClientObjs
+  
+  def initialize(clients,loop_interv=10)
+    case clients.class
+    when String
+      @cach = MemCache.new(clients.to_a,:namespace=>'caichen')
+    when Array
+      clientsx = clients.compact.delete_if{|x|x=='' || x.class != String}
+      @cach = MemCache.new(clientsx,:namespace=>'caichen')
+    end
+    init_config
+  end
+  
+  def init_config
+
+=begin rdoc
+  $HostClientObjs => :
+  {'http://10.2.226.20:9876' =>
+    {
+      :proc => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876'>,
+      :process => {
+        "123" => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876' pid=123>,
+        "456" => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876' pid=456>,
+        "789" => #<RemoteLmp::ProceAgent::baseUrl:'http://10.2.226.20:9876' pid=789>
+      }
+    }
+  }
+=end
+
+=begin rdoc
+  $HOSTS => :
+  [{
+    :ip => 'http://10.2.226.20:9876'
+    },{
+    :ip => 'http://10.2.226.37',
+    :pids => %w[123,456,789]
+  }]
+=end
+    File.open( File.join(File.dirname(__FILE__),'..','config','hosts.yaml') ) { |yf| @HOSTS = YAML::load( yf ) }
+    if @HOSTS
+      @HOSTS.each { |host|@HostClientObjs[host[:ip]] = {:proc => RemoteLmp::ProcAgent.new(host[:ip]),:process => (host[:pids]==nil ? [] : host[:pids].inject([]) { |arr, var| arr << RemoteLmp::ProcessAgent.new(host[:ip],var)  })}  }
+    end
+    @HOSTS
+  end
+  
+end
+
+
+class CaichenAgent
+  
+  def initialize(options={:processAgent => [],:interv => 10})
+    # options={:xiaoqian_ip,:cache,:procAgent,:processAgent=>[]}
+    @ip = options[:xiaoqian_ip] || options[:ip]
+    @procAgent = options[:procAgent]
+    @processAgent = options[:processAgent]
+    @cache = options[:cache]
+    @loop_interv = (options[:interv].to_i > 0 ? options[:interv].to_i : 10)
+    
+    # @INTR keep loop_job work until process get SIGINT.
+    @INTR = true
+    raise CaichenAgentInitError unless (@ip && @procAgent && @processAgent && @cache)
+  end
+  
+  def ext_response(bool,msg='OK')
+
+=begin rdoc
+  {
+      "success":true, // note this is Boolean, not string
+      "msg":"Consignment updated"
+  }
+=end
+    (bool == true) || (bool == 'true') ? "{\"success\":true,\"msg\":\"#{msg}}\"" : "{\"success\":false,\"msg\":\"#{msg}\"}"
+  end
+  
+  def get_hosts_grid_store_data
+    tmp = {}
+    t = Thread.fork{
+      tmp.update({:cpu => @procAgent.cpu["cpu"]})
+    }
+    tmp.update({
+      :ip => @procAgent.baseUrl,
+      :hostname => @procAgent.hostname,
+      :loadavg => @procAgent.loadavg["load"][0],
+      :mem_total => @procAgent.meminfo["MemTotal"].chomp(' kB'),
+      :mem_free => @procAgent.meminfo["MemFree"].chomp(' kB'),
+      :uptime_total => @procAgent.uptime["total"],
+      :uptime_idle => @procAgent.uptime["idle"],
+      :disk_size => @procAgent.df["size"],
+      :disk_use => @procAgent.df["use"],
+      :date => @procAgent.date
+    })
+    t.join
+    tmp
+  end
+  
+  def get_host_process_grid_store_data
+    # attributes = [:cwd,:exe,:root,:cmdline,:mem]
+    r = []
+    t = []
+    @processAgent.each { |key,obj|
+      tmp = {}
+      t << Thread.fork{
+        tmp.update({:cpu => obj.cpu})
+      }
+      tmp.update({
+        :pid => key, # obj[:process].pid,
+        :cwd => obj.cwd,
+        :exe => obj.exe,
+        :root => obj.root,
+        :cmdline => obj.cmdline,
+        :memory => obj.mem
+      })
+      r << tmp
+    }
+    t.each { |e| e.join  }
+    r
+  end
+  
+  def cpu
+    @procAgent.cpu["cpu"]
+  end
+  
+  def loadavg
+    @procAgent.loadavg["load"][0]
+  end
+  
+  def meminfo
+    @procAgent.meminfo.inject([]){|a,e|
+      a << {:type => e[0],:count => e[1]}
+    }
+  end
+  
+  def df
+    @procAgent.df.inject([]){|a,e|
+      a << {:type => e[0],:count => e[1]}
+    }
+  end
+  
+  def loop_job
+=begin rdoc
+  kill -2 <pid>
+  
+  Some of the more commonly used signals:
+       1       HUP (hang up)
+       2       INT (interrupt)
+       3       QUIT (quit)
+       6       ABRT (abort)
+       9       KILL (non-catchable, non-ignorable kill)
+       14      ALRM (alarm clock)
+       15      TERM (software termination signal)
+=end
+    Signal.trap( "SIGINT" ){
+      @INTR = false
+      p 'process got SIGINT,and it will stop loop_job'
+    }
+    # @INTR keep loop_job work until process get SIGINT.
+    while @INTR
+      st = Time.now.to_i
+      # do some thing
+      work_job
+      ed = Time.now.to_i
+      sleep (@loop_interv-(ed-st)) if (ed-st) > 0
+    end
+  end
+
+  def work_job
+=begin rdoc
+  we appoint that in memcache:
+  key,value
+  @ip+'_df',df
+  @ip+'_df' is key,and df is value.
+  so caichen could get the value by the key in cache.
+=end    
+    begin
+      @cache.set(@ip+'_df',df)
+      @cache.set(@ip+'_meminfo',meminfo)
+      @cache.set(@ip+'_loadavg',loadavg)
+      @cache.set(@ip+'_cpu',cpu)
+      @cache.set(@ip+'_get_host_process_grid_store_data',get_host_process_grid_store_data)
+      @cache.set(@ip+'_get_hosts_grid_store_data',get_hosts_grid_store_data)
+    rescue Exception => e
+      $Log.error "@cache.set error. @cache => #{@cache.inspect}. CaichenAgent(self) => #{self.inspect}"
+    end
+  end
+end
+
+
+
+
+
+
+
+
+
+
+

data/lib/bg_pull/lpmp.rb

@@ -0,0 +1,45 @@
+require 'rubygems'
+require "pp"
+require File.join(File.dirname(__FILE__),'remote_lmp.rb')
+require File.join(File.dirname(__FILE__),'utils.rb')
+
+begin
+  require 'sinatra/base'
+  require "memcache"
+  require "log4r"
+  include Log4r
+  require 'activesupport' unless ({}.respond_to? :to_json)
+rescue LoadError => e
+  p <<DOC
+-----------------------------------------------------------------------
+LPM say:
+The LPM server require sinatra and activesupport.
+Please exec "gem install sinatra" or "gem install activesupport" first.
+"gem install memcache" or "gem install log4r" as well.
+-----------------------------------------------------------------------
+Ruby say:
+#{e}
+-----------------------------------------------------------------------
+DOC
+  exit(-1)
+end
+
+$Log = Logger.new File.join(File.dirname(__FILE__),'log','lpmp2nd.log')
+File.open( File.join(File.dirname(__FILE__),'config','log.yaml') ) { |yf| $Log.level = YAML::load( yf ) }
+
+=begin rdoc
+  caichen = Caichen.new(...)
+  
+  caichenAgentPids = []
+  caichen.HostClientObjs.each do |ip,obj|
+    caichenAgentPids << Process.fork do
+      # @ip = options[:xiaoqian_ip] || options[:ip]
+      # @procAgent = options[:procAgent]
+      # @processAgent = options[:processAgent]
+      # @cache = options[:cache]
+      caichenagent = CaichenAgent.new({:ip=>ip,:procAgent=>obj[:proc],:processAgent=>ojb[:process],:cache=>caichen.cache})
+      caichenagent.work_job
+    end
+    Process.detach caichenAgentPids[-1]
+  end
+=end