fluent-plugin-dstat 0.1.0 → 0.2.0

data/Gemfile

@@ -7,9 +7,8 @@ source "http://rubygems.org"
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
+  gem "bundler", "~> 1.0"
   gem "jeweler", "~> 1.6.4"
-  gem "rcov", ">= 0"
 end
 
 gem "fluentd", "~> 0.10.7"

data/README.md

@@ -0,0 +1,46 @@
+Dstat plugin for Fluent
+
+Description goes here.
+
+## What's Dstat?
+
+Dstat is a versatile replacement for vmstat, iostat, netstat and ifstat.
+If you need more detail, see here[http://dag.wieers.com/home-made/dstat]
+This plugin use Dstat, so you need to install Dstat before using this plugin.
+
+## Configuration
+
+```
+<source>
+  type dstat
+  tag dstat
+  option -c
+  delay 3
+ </source>
+```
+
+* option:option for dstat command(default: -fcdnm)
+
+## Output Format
+
+When you use option -a, you get structured output data like below.
+
+  {
+  "hostname":"tsukuba000",
+   dstat":{"total cpu usage":"usr":"0.0","sys":"0.0","idl":"100.0","wai":"0.0","hiq":"0.0","siq":"0.0"},
+           "dsk/total":{"read":"0.0","writ":"0.0"},"net/total":{"recv":"148.0","send":"164.0"},
+           "paging":{"in":"0.0","out":"0.0"},
+           "system":{"int":"16.333","csw":"29.0"}}
+  }
+
+## Supported options
+
+```
+aio, cpu, cpu24, disk, epoch, fs, int, int24, io, ipc, load, lock, mem, net, page, page24, proc, raw, socket, swap, swapold, sys, tcp, udp, unix, vm, disk-util, freespace, top-bio, top-cpu,top-io, top-mem, top-oom, utmp, top-io -fc
+```
+
+## Copyright
+
+Copyright (c) 2011 Shunsuke Mikami. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -30,14 +30,6 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
-end
-
 task :default => :test
 
 require 'rdoc/task'

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/fluent-plugin-dstat.gemspec

@@ -5,22 +5,22 @@
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-dstat"
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Shunsuke Mikami"]
-  s.date = "2012-01-12"
+  s.date = "2012-07-29"
   s.email = "shun0102@gmail.com"
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
     "AUTHORS",
     "Gemfile",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "fluent-plugin-dstat.gemspec",
@@ -30,7 +30,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/shun0102/fluent-plugin-dstat"
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
+  s.rubygems_version = "1.8.23"
   s.summary = "Dstat Input plugin for Fluent event collector"
 
   if s.respond_to? :specification_version then
@@ -40,24 +40,21 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.7"])
       s.add_runtime_dependency(%q<rdoc>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
     else
       s.add_dependency(%q<fluentd>, ["~> 0.10.7"])
       s.add_dependency(%q<rdoc>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
     s.add_dependency(%q<fluentd>, ["~> 0.10.7"])
     s.add_dependency(%q<rdoc>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
   end
 end
 

data/lib/fluent/plugin/in_dstat.rb

@@ -2,123 +2,129 @@ module Fluent
 
 class DstatInput < Input
 
-  def split_second_key(str)
-    result = []
-    while (str)
-      index = /[^ ] / =~ str
-      if index
-        result << str[0..index]
-      else
-        result << str unless str == " "
-	return result
-      end
-      str = str[(index + 1)..-1]
-    end
-
-    return result
-  end
-
   Plugin.register_input('dstat', self)
 
   def initialize
     super
+    require 'csv'
     @hostname = `hostname -s`.chomp!
     @line_number = 0
     @first_keys = []
     @second_keys = []
+    @data_array = []
+    @max_lines = 100
   end
 
   config_param :tag, :string
   config_param :option, :string, :default => "-fcdnm"
   config_param :delay, :integer, :default => 1
+  config_param :tmp_file, :string, :default => "/tmp/dstat.csv"
 
   def configure(conf)
     super
-    @command = "dstat #{@option} #{@delay}"
+    @command = "dstat #{@option} --output #{@tmp_file} #{@delay}"
   end
 
   def start
+    if File.exist?(@tmp_file)
+      File.truncate(@tmp_file, 0)
+    else
+      `touch #{@tmp_file}`
+    end
     @io = IO.popen(@command, "r")
     @pid = @io.pid
+
+    @loop = Coolio::Loop.new
+    @dw = DstatCSVWatcher.new(@tmp_file, &method(:receive_lines))
+    @dw.attach(@loop)
     @thread = Thread.new(&method(:run))
   end
 
   def shutdown
     Process.kill(:TERM, @pid)
-    if @thread.join(60)  # TODO wait time
-      return
-    end
-    Process.kill(:KILL, @pid)
+    @dw.detach
+    @loop.stop
     @thread.join
+    File.delete(@tmp_file)
   end
 
   def run
-    @io.each_line(&method(:each_line))
-  end
-
-  private
-  def each_line(line)
     begin
-      line.chomp!
-      line.gsub!(/\e\[7l/, "")
+      @loop.run
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+    end
+  end
 
+  def receive_lines(lines)
+    lines.each do |line|
+      next if line == ""
       case @line_number
-      when 0
-        @first_keys = line.split(" ")
-      when 1
-        index = 0
-        @first_keys.each do |i|
-          @second_keys << line[index..(index + i.length - 1)]
-          index += i.length + 1
+      when 0..1
+      when 2
+        line.delete!("\"")
+        @first_keys = CSV.parse_line(line)
+        pre_key = ""
+        @first_keys.each_with_index do |key, index|
+          if key.nil? || key == ""
+            @first_keys[index] = pre_key
+          end
+          pre_key = @first_keys[index]
         end
-      else
-        hash = Hash.new()
-        values = []
-        index = 0
-        @first_keys.each do |i|
-          values << line[index..(index + i.length - 1)]
-          index += i.length + 1
+      when 3
+        line.delete!("\"")
+        @second_keys = line.split(',')
+        @first_keys.each_with_index do |key, index|
+          @data_array[index] = {}
+          @data_array[index][:first] = key
+          @data_array[index][:second] = @second_keys[index]
         end
-
-        @first_keys.each_with_index do |i, index|
-          value_hash = Hash.new()
-          first = i.gsub(/^-+|-+$/, "")
-          length = i.length
-
-          if first == "most-expensive"
-            s_key = @second_keys[index].gsub(/^\s+|\s+$/, "")
-            value_hash[s_key] = values[index]
-          else
-            keys = split_second_key(@second_keys[index])
-            second_index = 0
-
-            keys.each do |i|
-              next_index = second_index + i.length - 1
-              value = values[index][second_index..next_index]
-              second_index += i.length + 1
-              value = value.gsub(/^\s+|\s+$/, "") if value
-              value_hash[i.gsub(/^\s+|\s+$/, "")] = value
-            end
-          end
-
-          if hash[first].nil?
-            hash[first] = value_hash
-          else
-            hash[first] = hash[first].merge(value_hash)
-          end
+      else
+        values = line.split(',')
+        data = Hash.new { |hash,key| hash[key] = Hash.new {} }
+        values.each_with_index do |v, index|
+          data[@first_keys[index]][@second_keys[index]] = v
         end
-
         record = {
           'hostname' => @hostname,
-          'dstat' => hash
+          'dstat' => data
         }
         Engine.emit(@tag, Engine.now, record)
       end
+
+      if (@line_number % @max_lines) == (@max_lines - 1)
+        @dw.detach
+        File.truncate(@tmp_file, 0)
+        @dw = DstatCSVWatcher.new(@tmp_file, &method(:receive_lines))
+        @dw.attach(@loop)
+      end
+
       @line_number += 1
+    end
 
-    rescue
-      $log.error "exec failed to emit", :error=>$!, :line=>line
-      $log.warn_backtrace $!.backtrace
+  end
+
+  class DstatCSVWatcher < Cool.io::StatWatcher
+    INTERVAL = 0.500
+    attr_accessor :previous, :cur
+
+    def initialize(path, &receive_lines)
+      super path, INTERVAL
+      @path = path
+      @io = File.open(path)
+      @pos = 0
+      @receive_lines = receive_lines
+    end
+
+    def on_change(prev, cur)
+      buffer = @io.read(cur.size - @pos)
+      @pos = cur.size
+      lines = []
+      while line = buffer.slice!(/.*?\n/m)
+        lines << line.chomp
+      end
+      @receive_lines.call(lines)
     end
   end
 end

data/test/plugin/test_in_dstat.rb

@@ -9,9 +9,9 @@ class DstatInputTest < Test::Unit::TestCase
              "epoch", "fs", "int", "int24", "io", "ipc", "load", "lock",
              "mem", "net", "page", "page24", "proc", "raw", "socket",
              "swap", "swapold", "sys", "tcp", "udp", "unix", "vm",
-             "disk-tps", "disk-util", "dstat-cpu", "dstat-ctxt", "dstat-mem", "freespace",
-             "top-bio", "top-childwait", "top-cpu","top-io",
-             "top-mem", "top-oom", "utmp", "top-io -fc"]
+             "disk-util", "freespace",
+             "top-bio", "top-cpu","top-io",
+             "top-mem", "top-oom", "top-io -fc"]
 
   CONFIG = %[
     tag dstat
@@ -31,7 +31,6 @@ class DstatInputTest < Test::Unit::TestCase
   def test_emit
 
     OPTIONS.each do |op|
-      #emit_with_conf(CONFIG)
       conf = "tag dstat\n option --#{op}\n delay 1"
       emit_with_conf(conf)
     end
@@ -46,6 +45,7 @@ class DstatInputTest < Test::Unit::TestCase
     end
 
     length = `dstat #{d.instance.option} #{d.instance.delay} 1`.split("\n")[0].split("\s").length
+    puts `dstat #{d.instance.option} #{d.instance.delay} 3`
 
     emits = d.emits
     assert_equal true, emits.length > 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dstat
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-12 00:00:00.000000000Z
+date: 2012-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &22315940 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 0.10.7
   type: :runtime
   prerelease: false
-  version_requirements: *22315940
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.7
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &22315460 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *22315460
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &22314980 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,21 +53,31 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *22314980
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &22314500 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :development
   prerelease: false
-  version_requirements: *22314500
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &22314020 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,31 +85,25 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *22314020
-- !ruby/object:Gem::Dependency
-  name: rcov
-  requirement: &22313540 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *22313540
+        version: 1.6.4
 description: 
 email: shun0102@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
-- README.rdoc
+- README.md
 files:
 - .document
 - AUTHORS
 - Gemfile
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - fluent-plugin-dstat.gemspec
@@ -110,7 +124,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3115706696286106192
+      hash: -4423186394386880393
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -119,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Dstat Input plugin for Fluent event collector

data/README.rdoc

@@ -1,43 +0,0 @@
-= Dstat plugin for Fluent
-
-Description goes here.
-
-== What's Dstat?
-
-Dstat is a versatile replacement for vmstat, iostat, netstat and ifstat.
-If you need more detail, see here[http://dag.wieers.com/home-made/dstat]
-This plugin use Dstat, so you need to install Dstat before using this plugin.
-
-== Configuration
-
-  <source>
-    type dstat
-    tag dstat
-    option -c 3
-  </source>
-
-* option:option for dstat command(default: -fcdnm 1)
-
-== Output Format
-
-When you use option -c, you get structured output data like below.
-
-  {
-  "hostname":"tsukuba000",
-  "dstat":{"total-cpu-usage":{"usr":"0",
-                              "sys":"0",
-                              "idl":"100",
-                              "wai":"0",
-                              "hiq":"0",
-                              "siq":"0"}}
-  }
-
-== Supported options
-
-  aio, cpu, cpu24, disk, epoch, fs, int, int24, io, ipc, load, lock, mem, net, page, page24, proc, raw, socket, swap, swapold, sys, tcp, udp, unix, vm, disk-tps, disk-util, dstat-cpu, dstat-ctxt, dstat-mem, freespace, top-bio, top-childwait, top-cpu,top-io, top-mem, top-oom, utmp, top-io -fc
-
-== Copyright
-
-Copyright (c) 2011 Shunsuke Mikami. See LICENSE.txt for
-further details.
-