gri 10.0.0

data/lib/gri/scheduler.rb

@@ -0,0 +1,68 @@
+module GRI
+  class Scheduler
+    attr_accessor :queue, :writers, :fake_descr_hash
+
+    def initialize loop, metrics
+      @loop = loop
+      @metrics = metrics
+      @loop.on_detach {process_queue}
+      @writers = []
+    end
+
+    def process_queue
+      while @loop.collectors.size < 5
+        host, options = queue.shift
+        break unless host
+        next if host =~ /^GRIMETRICS/
+
+        ts = options['type']
+        col_types = ts ? ts.split(',') : ['snmp']
+        for col_type in col_types
+          process1 col_type, host, options
+        end
+      end
+    end
+
+    def process1 col_type, host, options
+      collector = Collector.create(col_type, host, options,
+                                   @fake_descr_hash) {|records|
+        for writer in @writers
+          writer.write records
+        end
+        @metrics[:record_count] += records.size
+      }
+      if collector
+        #puts "#{collector.class} (#{col_type}): #{host}" if $debug
+        interval = (options['interval'] || 300).to_i
+        collector.interval = interval
+        collector.timeout = [90, interval].min
+        collector.on_error {@metrics[:error_count] += 1}
+        collector.on_retry {@metrics[:retry_count] += 1}
+        Log.info "[#{$$}] #{host}: collect #{col_type}"
+        begin
+          @loop.run if collector.sync?
+          @loop.attach collector
+        rescue SystemCallError
+          Log.error "#{host}: ERROR: #{$!}"
+          @loop.detach collector
+        end
+        @metrics[:run_count] += 1
+      end
+    end
+
+    def finalize
+      @writers.each {|w| w.finalize if w.respond_to? :finalize}
+    end
+  end
+
+  class UScheduler < Scheduler
+    def process_queue
+      while @loop.collectors.size < 5
+        host, options = queue.shift
+        break unless host
+        @metrics[:nometrics] = 1 if host =~ /^GRIMETRICS/
+        process1 'tra', host, options
+      end
+    end
+  end
+end

data/lib/gri/sgraph.rb

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+module GRI
+  class Sgraph
+    @height = 20
+
+    def initialize options
+      @options = options
+    end
+
+    BARS = [" ",  "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█" ]
+    def bar(val, unit, non_fullwidth_font = false)
+      n = (val.to_f/unit)
+      @height.times.map{|i|
+        x = n - (i * 8)
+        if x <= 0
+          BARS.first
+        else
+          bar_symbol = if x < 8
+                         BARS[x]
+                       else
+                         BARS.last
+                       end
+          bar_symbol += " " if non_fullwidth_font
+          bar_symbol
+        end
+      }
+    end
+
+    def render(json, summary, url = nil)
+      #@options = {:t=>'d', :non_fullwidth_font=>false}
+      @height = 16
+      rowss = json['data']
+      meta = json['meta']
+      step    = meta['step']
+      start_timestamp = meta["start"]
+      end_timestamp   = meta["end"].to_i
+      s = Time.at(start_timestamp).localtime.strftime("%Y-%m-%d %H:%M:%S")
+      e = Time.at(end_timestamp  ).localtime.strftime("%Y-%m-%d %H:%M:%S")
+
+      rowss.transpose.each_with_index {|rows, i|
+        #puts "i: #{i}"
+        max     = rows.flatten.compact.max
+
+        u0 = (max / @height)
+        ex0 = (Math.log(u0) / Math.log(10)).floor
+        unit = (u0 / 8 / (10**(ex0-1))).ceil * (10**(ex0-1))
+
+        max_val = unit*@height
+        #max_val = max
+        #unit    = max_val / (@height * 8).to_f
+
+        puts "    #{(url)}"
+        puts "    #{s} -"
+        puts "  #{meta['legend'][i].center(78)}"
+        render_graph rows, unit
+        render_x_axis_labels(rows, start_timestamp, step)
+        puts ""
+
+        #sums = summary.first.last
+        #puts "    #{sprintf("cur: %.1f  ave: %.1f  max: %.1f  min %.1f", *sums)}"
+      }
+    end
+
+    def render_graph rows, unit
+      result = []
+
+      rows.map{|row|
+        bar(row, unit, @options[:non_fullwidth_font])
+      }.transpose.reverse.each_with_index do |row, i|
+        i = (@height- i)
+        if i.even?
+          n = unit * i * 8
+          if n > 10
+            label = sprintf("%6s", to_scalestr(unit * i * 8))
+          else
+            label = sprintf("%6g", unit * i * 8)
+          end
+        else
+          label = ''
+        end
+        line = row.join
+        if color = @options[:color]
+          line = Term::ANSIColor.send(color, line)
+        end
+        result << "#{sprintf('%6s', label)}|#{line}|"
+      end
+      puts result.join("\n")
+    end
+
+    def to_scalestr v, base=1000
+      if v == nil or base == nil or base == 0
+        return(v.to_i == v ? v.to_i : v)
+      end
+      v = v.to_f
+      if v >= base ** 4
+        "%gT" % (v / (base ** 4))
+      elsif v >= base ** 3
+        "%gG" % (v / (base ** 3))
+      elsif v >= base ** 2
+        "%gM" % (v / (base ** 2))
+      elsif v >= base
+        "%gK" % (v / base)
+      else
+        v.to_s
+      end
+    end
+
+    def render_x_axis_labels(rows, start_timestamp, step)
+      tm = rows.size * step
+
+      x_axis_labels = rows.length.times.select{|n| n % 8 == 0}. map{|n|
+        t = Time.at(start_timestamp + (n * step)).localtime
+        sprintf("%-8s", to_axis_label(t, tm))
+      }.join
+      x_axis_arrows= rows.length.times.select{|n| n % 8 == 0}. map{|n|
+        " /      "
+      }.join
+
+      case 'show'#@options[:x_axis_label]
+      when "show"
+        puts sprintf("%6s%s", "", x_axis_arrows)
+        puts sprintf("%6s%s", "", x_axis_labels)
+      when "simple"
+        puts sprintf("%6s%s", "", x_axis_labels)
+      end
+    end
+
+    def to_axis_label t, tm
+      f = if tm <= 3600
+            '%M'
+          elsif tm <= 48*3600
+            '%H:%M'
+          elsif tm <= 8*24*3600
+            '%a'
+          elsif tm <= 35*24*3600
+            '%d'
+          else
+            '%m'
+          end
+      t.strftime f
+    end
+
+    TERMS = {'sh'=>['Hour', 3600], 's8h'=>['8Hours', 8*3600],
+      'd'=>['Day', 24*3600], 'w'=>['Week', 7*24*3600],
+      'm'=>['Month', 31*24*3600], 'y'=>['Year', 366*24*3600]}
+  end
+end

data/lib/gri/spark.rb

@@ -0,0 +1,94 @@
+require 'optparse'
+require 'uri'
+require 'open-uri'
+require 'yaml'
+
+require 'gri/q'
+require 'gri/gparams'
+require 'gri/sgraph'
+
+module GRI
+  class Spark
+    def run options={}
+      optparser = optparse options
+      optparser.parse!
+      exit unless (url_s = ARGV.shift)
+
+      url = URI.parse url_s
+      params = GParams.new
+      parse_query url.query, params
+      fix_params params, options
+      url.query = mk_query params
+      if options[:user]
+        if options[:pass]
+          http_basic_authentication = [options[:user], options[:pass]]
+        else
+          print "Password:"
+          system "stty -echo"
+          password = $stdin.gets.chop
+          system "stty echo"
+          http_basic_authentication = [options[:user], password]
+        end
+      end
+      obj = fetch url, http_basic_authentication
+      graph = Sgraph.new options
+      graph.render obj, {}, url
+    end
+
+    def fix_params params, options
+      etime = params['etime'].to_i
+      etime = (Time.now + etime).to_i if etime <= 0
+      if (t = options[:t]) and (v = Sgraph::TERMS[t])
+        stime = (stime = params['stime']) ? stime.to_i : -v[1]
+        stime = (etime + stime).to_i if stime <= 0
+      else
+        stime = (stime = params['stime']) ? stime.to_i : -28*3600
+        stime = (etime + stime).to_i if stime <= 0
+      end
+      params['stime'] = stime
+      params['etime'] = etime
+      params['maxrows'] = 70
+      params['fmt'] = 'json'
+    end
+
+    def fetch url, http_basic_authentication
+      if http_basic_authentication
+        str = open(url,
+                   :http_basic_authentication=>http_basic_authentication).read
+      else
+        str = open(url).read
+      end
+      obj = YAML.load str
+      obj
+    end
+
+    def mk_query params
+      params.map {|k, v|
+        (Array === v) ? v.map {|vv| "#{k}=#{vv}"} : "#{k}=#{v}"
+      }.flatten.join('&')
+    end
+
+    def parse_query qs, params={}
+      (qs || '').split(/[&;] */n).each {|item|
+        k, v = item.split('=', 2)
+        params[k] = v
+      }
+      params
+    end
+
+    def optparse opts
+      op = OptionParser.new
+      op.on('--debug') {$debug = true; STDOUT.sync = true;
+        opts['log-level'] = 'debug'}
+      op.on('--Doption=STR') {|arg| (opts['Doption'] ||= []).push arg}
+      op.on('-O OPT_STR') {|arg| (opts['O'] ||= []).push arg}
+      op.on('-c', '--config-path=PATH') {|arg| opts[:config_path] = arg}
+      op.on('--log-level=LEVEL') {|arg| opts['log-level'] = arg}
+      op.on('--nop') {opts['nop'] = true}
+      op.on('-t ARG') {|arg| opts[:t] = arg}
+      op.on('-u', '--user=USER') {|arg| opts[:user] = arg}
+      op.on('-p', '--password=PASSWORD') {|arg| opts[:pass] = arg}
+      op
+    end
+  end
+end

data/lib/gri/tra_collector.rb

@@ -0,0 +1,80 @@
+require 'gri/collector'
+require 'gri/ldb'
+
+module GRI
+  class TraCollector < Collector
+    TYPES['tra'] = self
+
+    def on_attach
+      # @loop.detach self if @config['nop']
+      dir = "#{self.class.gra_dir}/#{host}"
+      Dir.mkdir dir unless File.directory? dir
+
+      db_class = self.class.db_class
+      db = (db_class <= RemoteLDB) ? db_class.new(self.class.tra_uri, host) :
+        db_class.new("#{self.class.tra_dir}/#{host}")
+
+      @loop.next_tick {
+        begin
+          update_rrd_dir dir, db, options
+        ensure
+          @loop.detach self
+          db.close
+        end
+      }
+    end
+
+    def update_rrd_dir dir, db, options
+      data_names = db.get_data_names
+      for data_name, interval in data_names
+        lu_path = "#{dir}/.lu_#{data_name}"
+        lu_time = nil
+        lu_pos = nil
+        if File.exist? lu_path
+          str = File.read(lu_path)
+          if str =~ /\A(\d+) (\d+)\Z/
+            lu_time = Time.at $1.to_i
+            lu_pos = $2.to_i
+          end
+        end
+        lu_time ||= Time.at(0)
+        lu_pos ||= 0
+
+        update_p = false
+        t = nil
+        records = []
+        db.get_after(data_name, lu_time, lu_pos) {|t, record, pos|
+          if $debug
+            ts = t.strftime '%Y-%m-%d %H:%M:%S'
+            puts "update #{ts} #{record['_host']} #{record['_key']}"
+          end
+          record['_interval'] = interval
+          records.push record
+          update_p = true
+          lu_time = t
+          lu_pos = pos
+          if records.size > 2000
+            @cb.call records
+            records.clear
+          end
+        }
+        if update_p
+          @cb.call records
+          open(lu_path, 'w') {|f| f.print "#{lu_time.to_i} #{lu_pos}"}
+        end
+      end
+    end
+
+    def recdump dir, records
+      path = dir + '/.rdump'
+      open(path, 'a') {|f|
+        records.each {|r| f.puts r.inspect}
+      }
+    end
+
+    class <<self
+      attr_accessor :tra_dir, :gra_dir
+      attr_accessor :db_class, :tra_uri
+    end
+  end
+end

data/lib/gri/trad.rb

@@ -0,0 +1,170 @@
+require 'gri/ldb'
+
+module GRI
+  class Trad
+    def initialize
+      @acls = nil
+      @tra_dir = nil
+    end
+
+    def public_dir
+      '/notexist'
+    end
+
+    def load_acls config
+      acls = config.getvar 'acl-permit'
+      acls = acls ? acls.map {|pat| Regexp.new pat} : []
+      acls.push(/^127\.0\.0\.1$/)
+      acls.push(/^::ffff:127\.0\.0\.1$/)
+      acls.push(/^::1$/)
+      acls
+    end
+
+    def allowed? acls, remote_addr
+      acls.detect {|re| remote_addr =~ re}
+    end
+
+    def call env
+      @acls ||= load_acls Config
+      unless allowed?(@acls, env['REMOTE_ADDR'])
+        return [401, {}, ['Unauthorized']]
+      end
+
+      @root_dir ||= Config['root-dir'] || Config::ROOT_PATH
+      @tra_dir ||= (Config['tra-dir'] || @root_dir + '/tra')
+
+      req = Rack::Request.new env
+      serve @tra_dir, req.path_info, req
+    end
+
+    def serve tra_dir, path_info, params
+      headers = {}
+      body = []
+
+      if path_info =~ /\A\/get\b/ #/
+        s = params['s']
+        dir = "#{tra_dir}/#{params['h']}"
+        if File.directory?(dir)
+          t = Time.at(params['t'].to_i)
+          pos = params['pos'].to_i
+          ldb = LocalLDB.new dir
+          headers.clear
+          prev_time = 0
+          ldb.getl_after(s, t, pos) {|time, line, pos|
+            if body.size > 1000 and time.to_i > prev_time
+              headers['X-GRI-Continue'] = "#{prev_time.to_s} #{pos}"
+              break
+            end
+            prev_time = time.to_i
+            body.push(line+"\n")
+          }
+          headers['X-GRI-Pos'] = pos.to_s
+        end
+      elsif path_info =~ /\A\/get_data_names\b/ #/
+        dir = "#{tra_dir}/#{params['h']}"
+        ldb = LocalLDB.new dir
+        ldb.get_data_names.each {|k, v| body.push "#{k}_#{v}\n"}
+      elsif path_info =~ /\A\/gritab\b/ #/
+        gritab_path = Config['gritab-path'] || @root_dir + '/gritab'
+        if File.exist? gritab_path
+          open(gritab_path) {|f|
+            while line = f.gets
+              next if line =~ /\A\s*#|\A\z/
+              body.push line
+            end
+          }
+        end
+      end
+      [200, headers, body]
+    end
+
+    def run options={}
+      optparser = optparse options
+      optparser.parse!
+      Process.daemon true if options[:daemonize] and !$debug
+      config_path = options[:config_path] || GRI::Config::DEFAULT_PATH
+      config = GRI::Config.init config_path
+      @acls ||= load_acls config
+      @root_dir ||= config['root-dir'] || GRI::Config::ROOT_PATH
+      @tra_dir ||= (config['tra-dir'] || @root_dir + '/tra')
+      log_dir = config['log-dir'] || @root_dir + '/log'
+      Dir.mkdir log_dir unless File.exist? log_dir
+      Log.init "#{log_dir}/#{optparser.program_name}.log"
+
+      bind_address = options[:bind_address] || '0.0.0.0'
+      port = options[:port] || 7079
+      server_sock = TCPServer.new bind_address, port
+      rs0 = [server_sock]
+      params = {}
+      while true
+        next unless (a = IO.select(rs0, nil, nil, 1))
+        rs, = a
+        for io in rs
+          begin
+            if io.kind_of? TCPServer
+              sock = server_sock.accept
+              peername = sock.peeraddr[2]
+              peeraddr = sock.peeraddr[3]
+              if allowed?(@acls, peeraddr)
+                puts "#{peeraddr}: accespt #{sock.object_id}" if $debug
+              else
+                sock.close
+                puts "#{peeraddr}: reject" if $debug
+                next
+              end
+              rs0.push sock
+            elsif io.eof?
+              io.close
+              rs0.delete io
+            else
+              line = io.gets
+              line.chomp!
+              if line =~ /\A(\/\w+)\s*(\S+)?$/ #/
+                pi = $1
+                qs = $2
+                params.clear
+                (qs || '').split(/[&;] */n).each {|item|
+                  k, v = item.split('=', 2)
+                  params[k] = v
+                }
+                puts "#{io.object_id}: serve #{pi} #{params.inspect}" if $debug
+                code, h, body = serve @tra_dir, pi, params
+                h.each {|k, v| io.puts "#{k}: #{v}"}
+                io.puts
+                body.each {|l|
+                  if l =~ /\A\./
+                    io.puts ".#{l}"
+                  else
+                    io.puts l
+                  end
+                }
+                io.puts "."
+              end
+            end
+          rescue Exception
+            Log.error "ERROR: #{$!}: #{$@.inspect}"
+          end
+        end
+      end
+    end
+
+    def optparse opts
+      op = OptionParser.new
+      op.on('--debug') {$debug = true; STDOUT.sync = true}
+      op.on('-c', '--config-path=PATH') {|arg| opts[:config_path] = arg}
+      op.on('-d', '--daemonize') {opts[:daemonize] = true}
+      op.on('-b', '--bind-address=ADDRESS') {|arg| opts[:bind_address] = arg}
+      op.on('-p', '--port=PORT', Integer) {|arg| opts[:port] = arg}
+    end
+
+    if __FILE__ == $0
+      require 'socket'
+      require 'optparse'
+      require 'gri/q'
+      require 'gri/config'
+      require 'gri/log'
+      require 'gri/util_daemon'
+      new.run
+    end
+  end
+end