gri 10.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b4c84175fc709e40d212ba2e061702b661b7af37
+  data.tar.gz: da221f34b501545a97daa79da88a8432c22adbd2
+SHA512:
+  metadata.gz: c445fdeeb8c6bf84b6caedcafe31225b0c604173a2d4e6ca0a0fe5d79dde171d834c0d3016b3abee9e29ac1227ccb362f472936046e49aeabe55c3ede6b53af0
+  data.tar.gz: 9c67f041a8117dfd3d0c25ac1e17b64b0aae7473ed9ba632f3ed0e796bfc0c5f68c714b5a7d9ac8030177ddef9da3bd60edd74a07b44bc2ff560d8d423d1d190

data/.gitignore

@@ -0,0 +1,7 @@
+*~
+.ruby-version
+.lu_*
+test/.t
+coverage/
+pkg/
+tmp/

data/MIT-LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013, Internet Initiative Japan Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,132 @@
+## About
+
+GRI is a monitoring tool that collects various data from network
+devices such as routers, switches, and hosts, and creates a lot of
+graphs from the collected data.
+
+GRI uses rrdtool for data storage and visualization, and the framework
+is written in Ruby.
+
+## Quick start guide
+
+### Requirements
+
+ * Unix OS
+ * apache or other web server that supports cgi
+ * rrdtool (1.0 or later, 1.4 later is highly recommended)
+ * Ruby (1.8.7 or later, 2.0 later is highly recommended)
+ * rack gem
+
+### Installation
+
+ 1. Install rrdtool
+ 2. `gem install rack`, `gem install gri --no-ri --no-rdoc`
+ 3. Copy the grapher cgi script into the directory for CGI executables used by your web server (commonly named cgi-bin). (e.g. `cp -p /usr/bin/grapher /var/www/cgi-bin`)
+
+### Configuration
+ 1. Create an administrative user for gri, e.g. "admin"
+ 2. Set up the gri root directory.
+<pre>
+# mkdir /usr/local/gri
+# chown admin /usr/local/gri
+</pre>
+Replace admin with the valid user that you created.
+ 3. Create `/usr/local/gri/gritab` file;
+gritab is a file used to specify the information collection target.
+<pre>
+host.example.com    ver=2c community=public
+router.example.com  ver=2c community=xxxxxxxx
+</pre>
+ 4. (Optional) Create `/usr/local/gri/gri.conf` file;
+gri.conf is a configuration file for setting GRI global parameters.
+<pre>
+root-dir    /usr/local/gri
+gritab-path /usr/local/gri/gritab
+font        DEFAULT:0:IPAPGothic
+</pre>
+ 5. Add a line to your /etc/crontab file similar to:
+<pre>
+*/5 * * * * admin /usr/bin/gri
+</pre>
+Replace admin with the valid user that you created.
+ 6. visit `http://<your host>/<path to>/grapher`, e.g. `http://localhost/cgi-bin/grapher`
+
+#### gritab
+
+gritab is a file used to specify the information collection target.
+By default, the file path is `/usr/local/gri/gritab`
+
+Example:
+<pre>
+# gritab example
+host.example.com    ver=2c community=public
+router.example.com  ver=2c community=xxxxxxxx
+</pre>
+
+ * A line beginning with # is a comment line.
+ * Each line consists of one host for which data is to be collected by GRI.
+ * The hosts can be specified in any order. They are SNMP-polled in the order specified.
+ * A host name can be followed by multiple options, each of which is separated by a space. When no option is specified, the default value is assumed.
+
+##### Description of typical options
+
+To disable (turn off) any of these options, specify `no-` at the beginning (example: `no-interfaces` stops collecting the interfaces MIB).
+
+* community=COMMUNITY
+
+  Specifies the text string password for SNMPv1 or SNMPv2c systems. (default: public)
+
+* interfaces
+
+  Specifies to get interfaces MIB. This option is enabled by default.
+
+* type=TYPE,...
+
+  Specifies the data collection type. When this option is omitted, snmp is assumed. Multiple sets can be specified with each set separated by a comma.
+
+* ver=VERSION
+
+  Specifies the snmp version supported by the agent. Available versions are "1" and "2c".
+
+#### gri.conf
+
+gri.conf is the settings file that determines the global operation of GRI.
+By default, the file path is `/usr/local/gri/gri.conf`.
+
+Example:
+<pre>
+# gri.conf example
+root-dir    /usr/local/gri
+gritab-path /usr/local/gri/gritab
+font        DEFAULT:0:IPAPGothic
+</pre>
+
+##### Description of typical options
+
+* root-dir *path*
+
+  Root directory of the GRI work directory [/usr/local/gri]
+
+* tra-dir *path*
+
+  tra directory [root-dir + /tra]
+
+* gra-dir *path*
+
+  gra directory [root-dir + /gra]
+
+* log-dir *path*
+
+  log directory [root-dir + /log]
+
+* plugin-dir *path*
+
+  plugin-dir [root-dir + /plugin]
+
+* option *option*
+
+  Option to be added to all gritab targets. Multiple options can be specified.
+
+* option-if-host *PAT* *option*
+
+  Option to be added to the default when the host name matches PAT (regexp). Multiple options can be specified.

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_helper"
+require "rake/testtask"
+
+Bundler::GemHelper.install_tasks :name=>'gri'
+Rake::TestTask.new

data/bin/grapher

@@ -0,0 +1,17 @@
+#! /usr/bin/ruby
+
+require 'rubygems'
+require 'rack'
+
+if File.symlink? __FILE__
+  org_path = File.dirname(File.expand_path(File.readlink(__FILE__)))
+else
+  org_path = File.dirname(File.expand_path(__FILE__))
+end
+$:.unshift org_path.untaint + '/../lib'
+
+require 'gri/grapher'
+
+wmain = GRI::WMain.new
+wmain.app = GRI::Grapher.new
+wmain.run

data/bin/gri

@@ -0,0 +1,5 @@
+#! /usr/bin/env ruby
+
+require 'gri/main'
+
+GRI::Main.new.run

data/bin/gricast

@@ -0,0 +1,17 @@
+#! /usr/bin/ruby
+
+require 'rubygems'
+require 'rack'
+
+if File.symlink? __FILE__
+  org_path = File.dirname(File.expand_path(File.readlink(__FILE__)))
+else
+  org_path = File.dirname(File.expand_path(__FILE__))
+end
+$:.unshift org_path.untaint + '/../lib'
+
+require 'gri/cast'
+
+wmain = GRI::WMain.new
+wmain.app = GRI::Cast.new
+wmain.run

data/bin/grispark

@@ -0,0 +1,5 @@
+#! /usr/bin/env ruby
+
+require 'gri/spark'
+
+GRI::Spark.new.run

data/bin/griwalk

@@ -0,0 +1,9 @@
+#! /usr/bin/env ruby
+
+require 'gri/main'
+require 'gri/app_walker'
+
+main = GRI::Main.new
+main.config['walker'] = true
+main.run
+

data/bin/trad

@@ -0,0 +1,19 @@
+#! /usr/bin/ruby
+
+if File.symlink? __FILE__
+  org_path = File.dirname(File.expand_path(File.readlink(__FILE__)))
+else
+  org_path = File.dirname(File.expand_path(__FILE__))
+end
+$:.unshift org_path.untaint + '/../lib'
+
+require 'socket'
+require 'optparse'
+
+require 'gri/config'
+require 'gri/log'
+require 'gri/q'
+require 'gri/util_daemon'
+require 'gri/trad'
+
+GRI::Trad.new.run

data/bin/trad-http

@@ -0,0 +1,17 @@
+#! /usr/bin/ruby
+
+require 'rubygems'
+require 'rack'
+
+if File.symlink? __FILE__
+  org_path = File.dirname(File.expand_path(File.readlink(__FILE__)))
+else
+  org_path = File.dirname(File.expand_path(__FILE__))
+end
+$:.unshift org_path.untaint + '/../lib'
+
+require 'gri/wmain'
+require 'gri/log'
+require 'gri/trad'
+
+GRI::WMain.new(GRI::Trad.new).run

data/gri.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "gri/version"
+
+Gem::Specification.new do |s|
+  s.name = "gri"
+  s.version = GRI::VERSION
+  s.authors = ["maebashi"]
+  s.homepage = ""
+  s.summary = %q{GRI}
+  s.description = %q{GRI}
+  s.files         = `git ls-files`.split("\n").select {|e| /^tmp/!~e}
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f
+) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+
+  s.add_runtime_dependency "rack"
+end

data/lib/fluent/plugin/out_gri.rb

@@ -0,0 +1,56 @@
+# coding: us-ascii
+
+require 'fileutils'
+
+if File.symlink? __FILE__
+  pdir = File.dirname(File.expand_path(File.readlink(__FILE__)))
+else
+  pdir = File.dirname(File.expand_path(__FILE__))
+end
+$LOAD_PATH.push pdir unless $LOAD_PATH.index pdir
+pdir = pdir + '/../..'
+$LOAD_PATH.push pdir unless $LOAD_PATH.index pdir
+
+require 'gri/log'
+require 'gri/builtindefs'
+require 'gri/msnmp'
+require 'gri/vendor'
+require 'gri/plugin'
+require 'gri/updater'
+require 'gri/rrd'
+require 'gri/ltsv'
+
+module Fluent
+
+class GriOutput < BufferedOutput
+  Plugin.register_output('gri', self)
+
+  config_param :gra_dir, :string, :default=>'/usr/local/gri/gra'
+
+  def configure conf
+    super
+  end
+
+  def start
+    super
+    #::Log.init '/tmp/fluent.log'
+    GRI::Plugin.load_plugins []
+    GRI::Config.init
+  end
+
+  def format tag, time, record
+    [tag, time, record].to_msgpack
+  end
+
+  def write chunk
+    records = []
+    chunk.msgpack_each {|tag, time, record|
+      records.push record
+    }
+    writer = GRI::Writer.create 'rrd', :gra_dir=>@gra_dir
+    writer.write records
+    writer.finalize
+  end
+end
+
+end

data/lib/gri/api.rb

@@ -0,0 +1,28 @@
+require 'fileutils'
+require 'gri/updater'
+
+module GRI
+  class API
+    def call env
+      if env['PATH_INFO'] =~ %r{^/api/(\w+)/(\w+)/(\w+)\b}
+        service_name, section_name, graph_name = $1, $2, $3
+        req = Rack::Request.new env
+        root_dir = Config['root-dir'] || Config::ROOT_PATH
+        cast_dir = root_dir + '/cast'
+        service_dir = "#{cast_dir}/#{service_name}"
+        host = section_name
+        key = "num_#{graph_name}"
+        FileUtils.mkdir_p service_dir
+
+        records = [{'_host'=>host, '_key'=>key, 'num'=>req['number']}]
+        writer = Writer.create 'rrd', :gra_dir=>service_dir, :interval=>60
+        writer.write records
+        writer.finalize
+        res = "OK\n"
+      else
+        res = "NG\n"
+      end
+      [200, {}, [res]]
+    end
+  end
+end

data/lib/gri/app_collector.rb

@@ -0,0 +1,154 @@
+require 'gri/config'
+require 'gri/scheduler'
+require 'gri/collector'
+require 'gri/loop'
+require 'gri/writer'
+require 'gri/q'
+
+module GRI
+  class AppCollector
+    attr_reader :config, :writers, :metrics
+
+    def initialize config
+      @config = config
+      @writers = []
+      @metrics = Hash.new 0
+    end
+
+    def run
+      start_time = Time.now
+      root_dir = config['root-dir'] ||= Config::ROOT_PATH
+
+      lines = load_target_lines config
+      targets = get_targets_from_lines lines, config
+
+      files = Config.getvar 'fake-descr-file'
+      fdh = load_fake_descr_files files if files
+
+      if config['updater']
+        if (tra_str = config['tra'])
+          tra_uri = get_tra_uri tra_str
+          TraCollector.tra_uri = tra_uri
+          TraCollector.db_class = RemoteLDB
+        else
+          tra_dir = config['tra-dir'] || root_dir + '/tra'
+          TraCollector.tra_dir = tra_dir
+          TraCollector.db_class = LocalLDB
+        end
+
+        gra_dir = config['gra-dir'] || root_dir + '/gra'
+        Dir.mkdir gra_dir unless File.directory? gra_dir
+        TraCollector.gra_dir = gra_dir
+        scheduler_class = UScheduler
+        h = {}
+        targets.each {|ary|
+          (hostname = ary[1]['hostname'] || ary[1]['alias']) and
+            (ary[0] = hostname)
+        }
+        targets = targets.select {|host, | f = h[host]; h[host] = true; !f}
+      else
+        scheduler_class = Scheduler
+      end
+
+      Log.info "START: pid #{$$}"
+      if config['para']
+        run_para targets, scheduler_class, start_time.to_i, fdh
+      else
+        run_single targets, scheduler_class, start_time.to_i, fdh
+      end
+      for writer in @writers
+        if writer.respond_to? :merge
+          writer.merge
+        end
+        if writer.respond_to? :purge_logs
+          writer.purge_logs
+        end
+      end
+      Log.info "END: pid #{$$}"
+      @metrics['targets'] = targets.size
+      @metrics['collector_elapsed'] = Time.now - start_time
+    end
+
+    def parse_host_key s
+      s.to_s.scan(/\A([-\.A-Za-z0-9]+)_([^_\d]*)(?:_?(.*))/).first
+    end
+
+    def load_fake_descr_files files
+      h = {}
+      for path in files
+        if File.exist? path
+          open(path) {|f|
+            while line = f.gets
+              if line =~ /\A([-\.\dA-Za-z]+_\S+)\s+(.*)/
+                descr = $2
+                host, key = $1.split(/_/, 2)
+                (h[host] ||= {})[key] = descr
+              end
+            end
+          }
+        end
+      end
+      h
+    end
+
+    def get_tra_uri tra_str
+      if tra_str =~ /\A[-\w\.]+(:\d+)\z/
+        tra_str = "http://#{tra_str}/"
+      elsif tra_str =~ /\A[-\w\.]+\z/
+        tra_str = "http://#{tra_str}:7080/"
+      end
+      uri = URI.parse tra_str rescue nil
+    end
+
+    def load_target_lines config
+      if config['updater'] and (tra_str = config['tra']) and
+          !config['gritab-path']
+        tra_uri = get_tra_uri tra_str
+        lines = RemoteLDB.get_gritab_lines tra_uri
+      else
+        root_dir = config['root-dir'] ||= Config::ROOT_PATH
+        gritab_path = config['gritab-path'] || root_dir + '/gritab'
+        lines = []
+        File.open(gritab_path) {|f|
+          while line = f.gets
+            lines.push line
+          end
+        }
+      end
+      lines
+    end
+
+    def get_targets_from_lines lines, config
+      targets = Config.get_targets_from_lines lines
+      goptions = Config.parse_options(*(config.getvar 'option'))
+      goptions.merge!(Config.parse_options(*config['O']))
+      if config['host-pat']
+        hosts_re = config.getvar('host-pat').map {|h| Regexp.new h}
+        targets = targets.select {|host, | hosts_re.detect {|re| re === host}}
+        #re = Regexp.new config['host-pat']
+        #targets = targets.select {|host, options| re === host}
+      end
+      for host, options in targets
+        hoptions = goptions.clone
+        hoptions.merge! Config.option_if_match(host, 'option-if-host', config)
+        hoptions.merge! options
+        options.replace hoptions
+      end
+      targets
+    end
+
+    def run_para targets, scheduler_class, start_time, fdh
+    end
+
+    def run_single targets, scheduler_class, start_time, fdh
+      loop = Loop.new
+      @writers.each {|writer| writer.loop = loop}
+      scheduler = scheduler_class.new loop, @metrics
+      scheduler.writers = @writers
+      scheduler.queue = targets
+      scheduler.process_queue
+      loop.run
+      scheduler.finalize
+    end
+  end
+end