gri 10.0.8 → 10.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19492a2ad3e4e1ec3c1e2f2f42de8cb962a3e591
-  data.tar.gz: 2c8951bbe68e3a24e8c1a3dbe5b024806f850a6d
+  metadata.gz: d0085633d9b1b963f804f959a9c2a17cc2b7d76f
+  data.tar.gz: cee9da876f1ab710bc5554230fa0c58e742d0979
 SHA512:
-  metadata.gz: 3f7b436519f3f84d94227d01a7ba9be89e9f1436614d191e9d2fe1360629b824a69fe5d6c0293314df1ffcf891c5411c4bc477ee5ced3506e2e3c23c279cbf6b
-  data.tar.gz: 01218f858aeefd19b5b83433072ee66cafdd7d19ed91b27716df8323d6319043df7f66c5d3bf879545d6847ad6cfcfe7980fec4455ef951bf619ede6c1a2901e
+  metadata.gz: 7b16f01fbde9f66c4f144b7dd1e0ad4437393d3db9657c882166b83ed3de47549a62094b34b1dbd5ae0307571c614b354441aeea072c21d40028935ddcf4be0c
+  data.tar.gz: d0134819ae4af120a405b8e6f2a930977fc13d3d77ed760c436c3a5800d415ed7e1a3fe1df819f3918527f15514275d5f7f25ff347407fbd40a27c007d03d92f

data/lib/fluent/plugin/out_gri.rb

@@ -58,4 +58,40 @@ class GriOutput < BufferedOutput
   end
 end
 
+class TraOutput < BufferedOutput
+  Plugin.register_output('tra', self)
+
+  config_param :config_path, :string, :default=>'/usr/local/gri/gri.conf'
+  config_param :tra_dir, :string, :default=>nil
+  config_param :log_level, :string, :default=>nil
+  config_param :interval, :integer, :default=>nil
+
+  def start
+    super
+    if @log_level
+      ::Log.init '/tmp/out_tra.log', :log_level=>@log_level
+    end
+    GRI::Config.init @config_path
+    root_dir = GRI::Config['root-dir'] ||= GRI::Config::ROOT_PATH
+    plugin_dirs = GRI::Config.getvar('plugin-dir') || [root_dir + '/plugin']
+    GRI::Plugin.load_plugins plugin_dirs
+    @tra_dir ||= GRI::Config['tra-dir'] || root_dir + '/tra'
+    @interval ||= (GRI::Config['interval'] || 300).to_i
+  end
+
+  def format tag, time, record
+    [tag, time, record].to_msgpack
+  end
+
+  def write chunk
+    records = []
+    chunk.msgpack_each {|tag, time, record|
+      records.push record
+    }
+    writer = GRI::Writer.create 'ldb', :tra_dir=>@tra_dir, :interval=>@interval
+    writer.write records
+    writer.finalize
+  end
+end
+
 end

data/lib/gri/builtindefs.rb

@@ -171,11 +171,16 @@ module GRI
       :tdb=>['hrStorage', 'hrStorageDescr', 'hrStorageSize', 'hrStorageUsed'],
       :fix_workhash=>lambda {|wh|
         if (th = wh[:hrStorageTotalHack])
-          ph = Hash[*(th.map {|k, v|
+          ht = Hash[*(th.map {|k, v|
                         [v['dskPath'],
                           (v['dskTotalHigh'].to_i * 4294967296 +
                            v['dskTotalLow'].to_i) * 1024]
                       }).flatten]
+          hu = Hash[*(th.map {|k, v|
+                        [v['dskPath'],
+                          (v['dskUsedHigh'].to_i * 4294967296 +
+                           v['dskUsedLow'].to_i) * 1024]
+                      }).flatten]
         end
         if (h = wh[:hrStorage])
           h.reject! {|k, v|
@@ -183,12 +188,16 @@ module GRI
           h.each {|k, v|
             u = v['hrStorageAllocationUnits'].to_i
             v.delete 'hrStorageType'
-            if ph and (t = ph[v['hrStorageDescr']])
+            if ht and (t = ht[v['hrStorageDescr']])
               v['hrStorageSize'] = t
             else
               v['hrStorageSize'] = v['hrStorageSize'].to_i * u
             end
-            v['hrStorageUsed'] = v['hrStorageUsed'].to_i * u
+            if hu and (t = hu[v['hrStorageDescr']])
+              v['hrStorageUsed'] = t
+            else
+              v['hrStorageUsed'] = v['hrStorageUsed'].to_i * u
+            end
           }
         end
       },
@@ -201,7 +210,8 @@ module GRI
     },
 
     'hrStorageTotalHack'=>{
-      :oid=>['dskPath', 'dskTotalLow', 'dskTotalHigh'],
+      :oid=>['dskPath', 'dskTotalLow', 'dskTotalHigh',
+        'dskUsedLow', 'dskUsedHigh'],
     },
 
     'hrSWRunPerf'=>{

data/lib/gri/collector.rb

@@ -99,6 +99,7 @@ module GRI
         if $debug
           puts "snmp: #{host} (#{vendor.name}): #{sysinfo['sysDescr']}"
           puts vendor.options.inspect
+          printf "  elapsed %f sec\n", Time.now - @attached_at
         end
 
         @snmp.version = vendor.options['ver'] if vendor.options['ver']
@@ -106,6 +107,7 @@ module GRI
         punits_d = punits.dup
         @workhash = punits.inject({}) {|h, pu| h[pu.cat] = {}; h}
         poll1(punits) {
+          printf "  elapsed %f sec\n", Time.now - @attached_at if $debug
           if !options.has_key?('SYS') or options['SYS']
             sysinfo.delete 'sysObjectID'
             records = [sysinfo]
@@ -168,8 +170,10 @@ module GRI
               records.push h
             end
           end
+          printf "  elapsed %f sec\n", Time.now - @attached_at if $debug
           @cb.call records
           @loop.detach self
+          printf "  elapsed %f sec\n", Time.now - @attached_at if $debug
         }
       }
     end
@@ -190,8 +194,16 @@ module GRI
         pu = punits.shift
         @get_oid_buf = []
         oids = pu.oids.dup
-        puts "  poll #{host} #{pu.name}" if $debug
-        walk1(pu, oids) {poll1 punits, &cb}
+        if $debug
+          t = Time.now
+          puts "  poll #{host} #{pu.name}" if $debug['walk']
+        end
+        walk1(pu, oids) {
+          if $debug and !$debug['walk']
+            printf "  poll #{host} #{pu.name} (%f sec)\n", Time.now - t
+          end
+          poll1 punits, &cb
+        }
       end
     end
 
@@ -250,10 +262,10 @@ module GRI
       @snmp.connect unless @snmp.sock
       @snmp.walk_start(enoid) {|*res| @results.push res}
       send_req enoid
-    rescue SocketError
+    rescue SocketError => e
       @loop.detach self
       @on_error.call if @on_error
-      Log.error "#{host}: error"
+      Log.error "#{host}: error: #{e.to_s}"
     end
 
     def get varbind, &cb
@@ -263,10 +275,10 @@ module GRI
       @snmp.connect unless @snmp.sock
       @snmp.get_start(varbind) {|*res| @results.push res}
       send_req varbind
-    rescue SocketError
+    rescue SocketError => e
       @loop.detach self
       @on_error.call if @on_error
-      Log.error "#{host}: error"
+      Log.error "#{host}: error: #{e.to_s}"
     end
 
     def send_req arg
@@ -290,8 +302,14 @@ module GRI
       elsif @preq
         Log.warn "#{host}: retry #{@retry_count}"
         @on_retry.call if @on_retry
-        @buffers.push @preq.last
-        loop.watch @snmp.sock, :w, @tout, self
+        if (s = @snmp.make_req(@snmp.state, @preq.last))
+          @buffers.push s
+          loop.watch @snmp.sock, :w, @tout, self
+        else
+          @loop.detach self
+          @on_error.call if @on_error
+          Log.error "#{host}: cannot retry: #{@retry_count}"
+        end
       else
         @loop.detach self
         @on_error.call if @on_error
@@ -314,8 +332,8 @@ module GRI
         @snmp.state = :SUCCESS if @snmp.state == :GET_REQ
       end
       if @snmp.state == :SUCCESS
-        @poll_cb.call @results
         @preq = nil
+        @poll_cb.call @results
       elsif @snmp.state != :IDLE
         send_req arg if arg
       end

data/lib/gri/main.rb

@@ -59,6 +59,7 @@ module GRI
         writer = Writer.create 'rrd', wopts
         app.writers.push writer
       end
+      app.writers.clear if @options['reset-writers']
       if @options['writers']
         for w in @options['writers']
           writer = Writer.create w, @config.to_h
@@ -103,6 +104,7 @@ module GRI
       op.on('--rrdupdater', '--updater') {opts['updater'] = true}
       op.on('--single') {opts['para'] = false}
       op.on('--tra=URL') {|arg| opts['tra'] = arg}
+      op.on('--reset-writers') {opts['reset-writers'] = true}
 
       op.on('-c COMMUNITY') {|arg| opts['community'] = arg}
       op.on('-v VER') {|arg| opts['version'] = arg}

data/lib/gri/page.rb

@@ -3,6 +3,7 @@ require 'gri/builtindefs'
 require 'gri/utils'
 require 'gri/request'
 require 'gri/format_helper'
+require 'gri/blank'
 
 module GRI
   class Page
@@ -15,10 +16,15 @@ module GRI
 
     def mk_page_title dirs, rs, params
       jstr = (params['p'] == 't' or params['p'] == 'v') ? ', ' : ' + '
-      hosthash = {}
       descrs = []
       headlegend = nil
 
+      if (tags = params.getvar('tag'))
+        title = tags.compact.join jstr
+        return title, title, headlegend
+      end
+
+      hosthash = {}
       for rname in rs
         host, key = rname.split('_', 2)
         hosthash[host] = true
@@ -52,7 +58,7 @@ module GRI
           descr
         end
       }.join(jstr)
-      body_title = descrs.collect {|url, xhost, descr|
+      body_title = descrs.map {|url, xhost, descr|
         if xhost and hosthash.size > 1
           "<a href=\"#{h url}\">#{h xhost}:#{h descr}</a>"
         else
@@ -66,6 +72,7 @@ module GRI
     def mk_param_str stime, etime, rs, ds, params
       res = rs.map {|r| "r=#{u r}"}
       res << "grp=#{u params['grp']}" if params['grp']
+      (tags = params.getvar('tag')) && tags.each {|tag| res << "tag=#{u tag}"}
       res << "stime=#{stime.to_i}"
       res << "etime=#{etime.to_i}" if etime.to_i.nonzero?
       res << "z=#{u params['z']}"
@@ -78,9 +85,15 @@ module GRI
 
     def mk_graph_tag stime, etime, rs, params
       if params['p'] == 't'
-        rs.map {|rname|
-          mk_graph_tag_r stime, etime, [rname], params
-        }.join("\n")
+        if (tags = params.getvar 'tag')
+          tags.map {|tag|
+            mk_graph_tag_r stime, etime, [], params
+          }.join("\n")
+        else
+          rs.map {|rname|
+            mk_graph_tag_r stime, etime, [rname], params
+          }.join("\n")
+        end
       else
         mk_graph_tag_r stime, etime, rs, params
       end
@@ -160,10 +173,18 @@ module GRI
       cs = stime.strftime '%Y-%m-%d %H:%M:%S'
       ce = etime.strftime '%Y-%m-%d %H:%M:%S'
 
-      r = params['r'] || ''
-      host, @data_name, index = parse_host_key r
-      rs = params.getvar 'r'
+      if (tags = params.getvar('tag')) and (grp = params['grp'])
+        gra_dir, = @options[:dirs]
+        rrdpaths = tags.inject([]) {|a, tag|
+          a += Dir.glob("#{gra_dir}/#{grp}/#{tag}/*.rrd")
+        }
+        r = File.basename rrdpaths.first
+      else
+        r = params['r'] || ''
+      end
+      host, @data_name, = parse_host_key r
 
+      rs = params.getvar('r') || []
       @title, body_title, headlegend =
         mk_page_title @options[:dirs], rs, params
 
@@ -190,6 +211,7 @@ module GRI
 <form enctype="application/x-www-form-urlencoded" method="get"
  action="<%= url_to '?' %>">
 <% rs.each {|r| -%><%= hidden 'r', r %><% } -%>
+<% tags and tags.each {|tag| -%><%= hidden 'tag', tag %><% } -%>
 <% if params['grp'] then %><%= hidden 'grp', params['grp'] %><% end %>
 <% tzs = TZS -%>
 <% (tzs.assoc(params['tz']) || tzs[0])[2] = true -%>
@@ -210,7 +232,7 @@ Graph size: <%= popup_menu('z', nil, *zs) %>
 <% (tms.assoc(params['tm']) || tms[1])[2] = true -%>
 term: <%= popup_menu('tm', nil, *tms) %>
 <br/>
-<% if rs.size > 1 -%>
+<% if rs.size > 1 or grp -%>
 <% c_ary = [['', 'sum'], ['s', 'stack'], ['v', 'overlay'], ['t', 'tile']] -%>
 <% (c_ary.assoc(params['p']) || c_ary[0])[2] = true -%>
 Composite type: <%= popup_menu('p', nil, *c_ary) %><br/>

data/lib/gri/pcollector.rb

@@ -66,7 +66,7 @@ module GRI
           pkeys.shift
           pts += ptargets[t]
           while (n = pts.shift)
-            timeout(waittime) {sock = server_sock.accept}
+            Timeout.timeout(waittime) {sock = server_sock.accept}
             if (res = IO.select(nil, [sock], nil, 20))
               thost = targets[n].first
               sock.puts "#{n} #{thost}"

data/lib/gri/scheduler.rb

@@ -25,9 +25,11 @@ module GRI
 
     def process1 col_type, host, options
       return if Config['nop']
+      return if col_type == 'fluentd'
       collector = Collector.create(col_type, host, options,
                                    @fake_descr_hash) {|records|
         for writer in @writers
+          puts "  writer #{writer.class}" if $debug
           writer.write records
         end
         @metrics[:record_count] += records.size

data/lib/gri/version.rb

@@ -1,3 +1,3 @@
 module GRI
-  VERSION = "10.0.8"
+  VERSION = "10.0.9"
 end

data/test/test_collector.rb

@@ -18,7 +18,7 @@ end
 
 class TestCollector < Test::Unit::TestCase
   TEST_SYS = "0\x81\xF2\x02\x01\x01\x04\bpublic00\xA2\x81\xE2\x02" +
-    "\x02\x03\xE9\x02\x01\x00\x02\x01\x000\x81\xD50d\x06\b+\x06\x01" +
+    "\x02\x03\xEA\x02\x01\x00\x02\x01\x000\x81\xD50d\x06\b+\x06\x01" +
     "\x02\x01\x01\x01\x00\x04XLinux xxxxx.yyyyy.zzz.com " +
     "X.X.XX-XXX.XX.X.XXX #1 SMP Thu Oct 31 12:00:00 EDT 2013 x86_64" +
     "0\x1F\x06\b+\x06\x01\x02\x01\x01\x05\x00\x04\x13xxxxx.yyyyy.zzz.com" +

data/test/test_docker_collector.rb

@@ -22,7 +22,8 @@ class TestDockerCollector < Test::Unit::TestCase
 
     s = '{"GitCommit":"28ff62e/0.7.9","GoVersion":"go1.2","KernelVersion":"2.6.32-999.9.abc.x86_64","Version":"0.7.9"}'
     collector.on_read s + "\n"
-
+  end
+  def test_dummy
   end
 end
 

data/test/test_format_helper.rb

@@ -30,7 +30,8 @@ class TestFormatHelper < Test::Unit::TestCase
   def test_format_helper_mk_query
     ae '?a=1', mk_query(:a=>1)
     assert_match /\bb=a\+%26\b/, mk_query(:a=>1, :b=>'a &')
-    assert_not_match /\bc=/, mk_query(:c=>nil)
+    #assert_not_match /\bc=/, mk_query(:c=>nil)
+    assert(/\bc=/ !~ mk_query(:c=>nil))
   end
 
   def test_format_helper_td
@@ -44,8 +45,10 @@ class TestFormatHelper < Test::Unit::TestCase
   end
 
   def test_format_helper_text_field
-    ae '<input type="text" name="text" value="aaa" size="40"/>',
-      text_field('text', 'aaa', 40, nil, nil)
+    t = text_field('text', 'aaa', 40, nil, nil)
+    s = t.scan(/\b(\w+="[^\"]+")/).flatten.sort
+    assert_equal ['name="text"', 'size="40"', 'type="text"', 'value="aaa"'], s
+    assert_match /<input\s+.*\/>/, t #/
   end
 
   def test_format_helper_popup_menu

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gri
 version: !ruby/object:Gem::Version
-  version: 10.0.8
+  version: 10.0.9
 platform: ruby
 authors:
 - maebashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-04 00:00:00.000000000 Z
+date: 2017-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: GRI