gri 10.0.0

data/lib/gri/page.rb

@@ -0,0 +1,235 @@
+require 'time'
+require 'gri/builtindefs'
+require 'gri/utils'
+require 'gri/request'
+require 'gri/format_helper'
+
+module GRI
+  class Page
+    include Utils
+    include FormatHelper
+
+    def initialize options={}
+      @options = options
+    end
+
+    def mk_page_title dirs, rs, params
+      jstr = (params['p'] == 't' or params['p'] == 'v') ? ', ' : ' + '
+      hosthash = {}
+      descrs = []
+      headlegend = nil
+
+      for rname in rs
+        host, key = rname.split('_', 2)
+        hosthash[host] = true
+        dir, records = search_records dirs, host
+        if records and (prop = get_prop records[key])
+          xhost = prop[:description] ? nil : host
+          descr = (prop[:description] || prop[:name] || '?').to_s
+          url = if params['grp'].blank?
+                  url_to("?r=#{rname}")
+                else
+                  url_to("?grp=#{params['grp']}&r=#{rname}")
+                end
+          descrs.push [url, xhost, descr]
+          unless headlegend or params['grp']
+            ub = prop[:ub]
+            if descr or ub
+              name = prop[:name].to_s
+              url = url_to("#{host}")
+              headlegend = "<a href=\"#{h url}\">#{h host}</a>"
+              headlegend << ' ' + name if descr
+              #headlegend << ", MAX: #{to_scalestr ub, 1000}"
+            end
+          end
+        end
+      end
+
+      html_title = descrs.map {|url, xhost, descr|
+        if xhost and hosthash.size > 1
+          xhost + ':' + descr
+        else
+          descr
+        end
+      }.join(jstr)
+      body_title = descrs.collect {|url, xhost, descr|
+        if xhost and hosthash.size > 1
+          "<a href=\"#{h url}\">#{h xhost}:#{h descr}</a>"
+        else
+          "<a href=\"#{h url}\">#{h descr}</a>"
+        end
+      }.join(jstr)
+
+      return html_title, body_title, headlegend
+    end
+
+    def mk_param_str stime, etime, rs, ds, params
+      res = rs.map {|r| "r=#{u r}"}
+      res << "grp=#{u params['grp']}" if params['grp']
+      res << "stime=#{stime.to_i}"
+      res << "etime=#{etime.to_i}" if etime.to_i.nonzero?
+      res << "z=#{u params['z']}"
+      res << "tz=#{u params['tz']}"
+      res << "y=#{u params['y']}"
+      res << "p=#{u params['p']}" if params['p']
+      res << "ds=#{ds}" if ds
+      res.join('&')
+    end
+
+    def mk_graph_tag stime, etime, rs, params
+      if params['p'] == 't'
+        rs.map {|rname|
+          mk_graph_tag_r stime, etime, [rname], params
+        }.join("\n")
+      else
+        mk_graph_tag_r stime, etime, rs, params
+      end
+    end
+
+    def mk_graph_tag_r stime, etime, rs, params
+      specs = DEFS.get_specs @data_name
+      if (gr_specs = specs[:graph]) and gr_specs.size >= 1
+        if params['p'] == 's' or params['p'] == 'v'
+          (0..gr_specs.size-1).map {|gidx|
+            gr_spec = gr_specs[gidx]
+            pickup_re = gr_spec[3]
+            dss = specs[:ds] || []
+            dss = dss.grep pickup_re if pickup_re
+            dss = dss.map {|ds| ds.split(',', 3)[1]}
+            dss.map {|ds|
+              param_str = mk_param_str stime, etime, rs, ds, params
+              mk_graph_tag_s param_str, gidx
+            }.join ''
+          }.join ''
+        else
+          param_str = mk_param_str stime, etime, rs, nil, params
+          (0..gr_specs.size-1).map {|gidx|
+            mk_graph_tag_s param_str, gidx}.join ''
+        end
+      end
+    end
+
+    def mk_graph_tag_s param_str, gidx=nil
+      param_str += "&g=#{gidx}" if gidx
+      name = ENV['SCRIPT_NAME'] || ''
+      "<a href=\"#{name}/#{h param_str}\">" +
+        "<img ismap src=\"#{name}?#{h param_str}\"></a><br/>"
+    end
+
+    def parse_tstr tstr
+      (tstr and !tstr.empty?) ? Time.parse(tstr) : nil
+    end
+
+    def parse_request req
+      now = Time.now
+      if @options[:clicked]
+        s = req.path_info[1..-1]
+        req.query_string = s
+        params = req.gparams
+        z = params['z']
+        width, = GRI::Graph::SIZE[z]
+
+        canvas_x_offset = (Config['canvas-x-offset'] || 75).to_i
+        imgx = @options[:imgx].to_i - canvas_x_offset
+        imgy = @options[:imgy].to_i
+
+        stime = params['stime'].to_i
+        etime = params['etime'].to_i
+        etime = (now + etime).to_i if etime <= 0
+        stime = (etime + stime).to_i if stime <= 0
+        stime = Time.at stime
+        time = stime + ((Time.at(etime) - stime) / width) * imgx
+        stime = Time.local(time.year, time.mon, time.day)
+        etime = stime + 24*3600
+        params['pt'] = 's'
+      else
+        params = req.gparams
+        deftime = now - now.to_i % 60
+        stime = parse_tstr(params['cs']) || deftime - 7*24*3600
+        etime = parse_tstr(params['ce']) || deftime
+      end
+      return stime, etime, params
+    end
+
+    def call env
+      req = GRI::Request.new env
+      ENV['TZ'] = req.params['tz'].to_s unless req.params['tz'].blank?
+      stime, etime, params = parse_request req
+
+      cs = stime.strftime '%Y-%m-%d %H:%M:%S'
+      ce = etime.strftime '%Y-%m-%d %H:%M:%S'
+
+      r = params['r'] || ''
+      host, @data_name, index = parse_host_key r
+      rs = params.getvar 'r'
+
+      @title, body_title, headlegend =
+        mk_page_title @options[:dirs], rs, params
+
+      defs_term = DEFS[:term]
+      sym = params['tm'].blank? ? '' : params['tm'].intern
+      terms = defs_term[sym] || defs_term[:default]
+      body = render(Grapher.layout) {render template, binding}
+      [200, {'Content-type' => 'text/html'}, [body]]
+    end
+
+    TZS = [['', 'localtime'], ['JST-9', 'JST-9'],
+      ['EST5EDT', 'EST5EDT'], ['PST8PDT', 'PST8PDT'],
+      ['Europe/London', 'Europe/London'],
+      ['Europe/Amsterdam', 'Europe/Amsterdam'],
+      ['Singapore', 'Singapore']]
+    def template
+      <<'EOS'
+<span class="large"><%= body_title %></span><br/>
+<% if params['z'] != 'll' and rs.size == 1 and headlegend and headlegend.size > 0 -%>
+<span class="small"><%= headlegend %></span><br/>
+<% end -%>
+<br/>
+
+<form enctype="application/x-www-form-urlencoded" method="get"
+ action="<%= url_to '?' %>">
+<% rs.each {|r| -%><%= hidden 'r', r %><% } -%>
+<% if params['grp'] then %><%= hidden 'grp', params['grp'] %><% end %>
+<% tzs = TZS -%>
+<% (tzs.assoc(params['tz']) || tzs[0])[2] = true -%>
+TIMEZONE: <%= popup_menu('tz', nil, *tzs) %>
+<% nflag, uflag = (params['y'] == 'u') ? [false, true] : [true, false] -%>
+Y-axis scale:
+<%= radio_button 'y', 'a', nflag %>auto
+<%= radio_button 'y', 'u', uflag %>upper limit<br/>
+<%= check_box 'pt', 's', (params['pt'] == 's') %>
+<nobr>
+from <%= text_field 'cs', cs, 20, 19, nil %>
+to <%= text_field 'ce', ce, 20, 19, nil %>
+</nobr>
+<% zs = [['ss', 'SS'], ['s', 'S'], ['m', 'M'], ['l', 'L'], ['ll', 'LL']] -%>
+<% (zs.assoc(params['z']) || zs[2])[2] = true -%>
+Graph size: <%= popup_menu('z', nil, *zs) %>
+<% tms = defs_term.sort_by {|k, v| v[0][1]}.map {|k,| [k.to_s, k.to_s]} -%>
+<% (tms.assoc(params['tm']) || tms[1])[2] = true -%>
+term: <%= popup_menu('tm', nil, *tms) %>
+<br/>
+<% if rs.size > 1 -%>
+<% c_ary = [['', 'sum'], ['s', 'stack'], ['v', 'overlay'], ['t', 'tile']] -%>
+<% (c_ary.assoc(params['p']) || c_ary[0])[2] = true -%>
+Composite type: <%= popup_menu('p', nil, *c_ary) %><br/>
+<% end -%>
+<input class="btn btn-primary btn-sm" type="submit" value="submit">
+</form>
+
+<% if params['pt'] == 's' -%>
+<%= mk_graph_tag stime.to_i, etime.to_i, rs, params %>
+<% else -%>
+<% for label, int in terms -%>
+<hr/><p>
+<strong><%=h label %> Graph</strong><br/>
+<%= mk_graph_tag -int, 0, rs, params %>
+</p>
+<% end -%>
+<% end -%>
+
+<hr/><!--%= RUBY_VERSION %-->
+EOS
+    end
+  end
+end

data/lib/gri/pcollector.rb

@@ -0,0 +1,209 @@
+require 'timeout'
+require 'gri/util_marshal'
+require 'gri/app_collector'
+
+module GRI
+  class AppCollector
+    def run_para targets, scheduler_class, start_time, fdh
+      sock_path = config['sock-path'] || '/tmp/.gcollectsock'
+      begin
+        server_sock = UNIXServer.new sock_path
+      rescue SystemCallError
+        puts "#{$!}: server_sock error" if $debug
+        Log.fatal "#{$!}: server_sock error"
+        return
+      end
+
+      duration = (config['duration'] || 0).to_i
+      if duration.zero?
+        basetime = start_time
+        offset = 0
+      else
+        basetime = start_time - start_time % duration
+        offset = start_time - basetime
+      end
+      interval = (config['interval'] || 300).to_i
+      ptargets = get_ptargets targets, basetime, duration, offset, interval
+      log_dir = config['log-dir'] || (config['root-dir'] + '/log')
+      Dir.glob("#{log_dir}/res.*.dump") {|path| File.unlink path} rescue nil
+      max_processes = (config['max-processes'] ||
+                       config['max-fork-process']).to_i
+      max_processes = 30 if max_processes < 1
+      nproc = [targets.size * 2 / 3 + 1, max_processes].min
+      waittime = [20, duration].min
+      begin
+        pids = fork_child server_sock, sock_path, nproc, targets, log_dir,
+          scheduler_class, fdh
+        server_loop targets, ptargets, server_sock, waittime
+      rescue TimeoutError, SystemCallError
+        Log.error $!.inspect
+      ensure
+        server_sock.close
+        Log.info "server_sock.close"
+        File.unlink sock_path
+      end
+      pids.each {|pid| Process.waitpid pid}
+      Dir.glob("#{log_dir}/res.#{$$}.*.dump") {|path|
+        begin
+          res = Marshal.load_from_file path
+          res.each {|k, v| @metrics[k] += v}
+          File.unlink path
+        rescue SystemCallError
+          Log.error "{$!}"
+        end
+      }
+    end
+
+    def server_loop targets, ptargets, server_sock, waittime
+      sock = nil
+      pkeys = ptargets.keys.sort
+      pts = []
+      while true
+        break if pkeys.empty?
+        now = Time.now.to_f
+        t = pkeys.first
+        if t <= now
+          pkeys.shift
+          pts += ptargets[t]
+          while (n = pts.shift)
+            timeout(waittime) {sock = server_sock.accept}
+            if (res = IO.select(nil, [sock], nil, 20))
+              thost = targets[n].first
+              sock.puts "#{n} #{thost}"
+              sock.close
+            else
+              sock.close
+              raise TimeoutError, 'select timeout'
+            end
+            if pts.empty? and (t = pkeys.first)
+              now = Time.now.to_f
+              if t <= now
+                pkeys.shift
+                small, big = [pts, ptargets[t]].sort_by {|e| e.size}
+                pts.replace big.zip(small)
+                pts.flatten!
+                pts.compact!
+                #pts += ptargets[t]
+              end
+            end
+          end
+        else
+          sleep(t - now)
+        end
+      end
+    end
+
+    def get_ptargets targets, basetime, duration, offset=0, default_interval=300
+      if duration.zero?
+        ptargets = {basetime=>(0..targets.size-1).to_a}
+      else
+        intervals = {}
+        n = 0
+        for host, options in targets
+          interval = (options['interval'] || default_interval).to_i
+          next if interval.zero?
+          (intervals[interval] ||= []).push n
+          n += 1
+        end
+        ptargets = {}
+        et = basetime + duration
+        for interval in intervals.keys
+          st = basetime - basetime % interval
+          (0..duration/interval).each {|n|
+            s = n * interval
+            if (t = st + s) >= basetime and t < et
+              ptargets[t+offset] ||= []
+              ptargets[t+offset] += intervals[interval]
+            end
+          }
+        end
+      end
+      ptargets
+    end
+
+    def get_max_queue_size
+      4
+    end
+
+    def fillup_queue n, sock_path, targets, scheduler
+      e = false
+      mqs = get_max_queue_size
+      while scheduler.queue.size < mqs
+        begin
+          unless File.socket? sock_path
+            e = true
+            break
+          end
+          sock = UNIXSocket.new sock_path
+        rescue Errno::ECONNREFUSED
+          sock.close rescue nil
+          sleep(0.1 + rand)
+          retry
+        rescue SystemCallError
+          sock.close rescue nil
+          e = true
+          break
+        end
+        begin
+          unless (line = sock.gets)
+            e = true
+            break
+          end
+        rescue
+          e = true
+          break
+        ensure
+          sock.close
+        end
+        num, host = line.split
+        scheduler.queue.push targets[num.to_i]
+        scheduler.process_queue
+      end
+      e
+    end
+
+    def fork_child server_sock, sock_path, nproc, targets, log_dir,
+        scheduler_class, fdh
+      pids = []
+      ppid = $$
+      for n in 1..nproc
+        pid = fork {
+          server_sock.close
+          start_time = Time.now
+          sleep 0.05 * n
+          Log.debug "child ##{n}"
+          loop = Loop.new
+          @writers.each {|writer| writer.loop = loop}
+          scheduler = scheduler_class.new loop, @metrics
+          scheduler.queue = []
+          scheduler.writers = @writers
+          scheduler.fake_descr_hash = fdh
+
+          e = fillup_queue n, sock_path, targets, scheduler
+          scheduler.process_queue
+          if !e or loop.has_active_watchers?
+            while true
+              loop.run_once
+              break if e and !loop.has_active_watchers?
+              e = fillup_queue n, sock_path, targets, scheduler
+              scheduler.process_queue
+            end
+          end
+          scheduler.finalize
+          rc = @metrics[:run_count]
+          elapsed = Time.now - start_time
+          Log.debug "end ##{n} #{rc} #{rc/elapsed}"
+          #@metrics["run_count#{n}".intern] = rc
+          begin
+            path = "#{log_dir}/res.#{ppid}.#{$$}.dump"
+            Marshal.dump_to_file @metrics, path
+          rescue SystemCallError
+            Log.error "#{$!}"
+          end
+        }
+        pids.push pid
+      end
+      pids
+    end
+  end
+end

data/lib/gri/plugin.rb

@@ -0,0 +1,75 @@
+module GRI
+  module Plugin
+    extend Plugin
+
+    def load_plugins dirs=[], config=nil
+      @loaded = {}
+      dirs += get_gem_dirs
+      dirs.push File.join(File.dirname(__FILE__), 'plugin')
+      dirs.each {|dir| load_plugin_dir dir, config}
+    end
+
+    def load_plugin_dir dir, config=nil
+      dir = File.expand_path dir
+      return unless File.exists? dir
+      files = get_plugin_files dir, config
+      files.each {|fname|
+        unless @loaded[fname]
+          path = File.join dir, fname
+          require path
+          @loaded[fname] = path
+        end
+      }
+    end
+
+    def get_plugin_files dir, config=nil
+      files = Dir.entries(dir).sort.select {|fname| fname =~ /\A[^.].*\.rb$/}
+      if config
+        if config['enable-plugin']
+          eps = config.getvar 'enable-plugin'
+          files = files.select {|fname|
+            s = fname.sub(/\.rb$/, '')
+            eps.detect {|pname| pname == s}
+          }
+        end
+        if config['disable-plugin']
+          dps = config.getvar 'disable-plugin'
+          files = files.select {|fname|
+            s = fname.sub(/\.rb$/, '')
+            !dps.detect {|pname| pname == s}
+          }
+        end
+      end
+      files
+    end
+
+    def get_gem_dirs
+      dirs = []
+      if Object.const_defined?(:Gem)
+        if defined?(::Gem::Specification) and
+            Gem::Specification.respond_to?(:find_all)
+          specs = Gem::Specification.find_all {|spec|
+            spec.full_require_paths.map {|path|
+              File.directory?(path + '/gri/plugin')}.any?
+          }.sort_by {|spec| spec.version}.reverse
+          names = {}
+          specs.each {|spec| names[spec.name] ||= spec}
+          names.values.each {|spec|
+            dirs += spec.full_require_paths.map {|path| path + '/gri/plugin'}
+          }
+        elsif Gem.respond_to?(:searcher)
+          specs = Gem.searcher.find_all 'gri/plugin/*.rb'
+          names = {}
+          specs.sort_by {|spec| spec.version}.reverse.each {|spec|
+            names[spec.name] ||= spec
+          }
+          names.values.each {|spec|
+            files = Gem.searcher.matching_files spec, 'gri/plugin/*.rb'
+            dirs += files.map {|fname| File.dirname fname}
+          }
+        end
+      end
+      dirs
+    end
+  end
+end