logstash-input-dstat 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ff2dc0530d6ad438d352678607f31c72f6d4bdff
+  data.tar.gz: 1db52882410bf63075093d74d961c1d62fb4aa9f
+SHA512:
+  metadata.gz: 4b0d583a077d1a21b0e01c65955b2f90dd5c1bc7d75fa2984c89516e8494bb93704cb6e399bd36d6a11765c4fd358b9b0f346c567e6f162c3d1739e0f38d4c28
+  data.tar.gz: f018bbb25388234e1f6968f595f99dec34a697f78973c502ed1fc52ab72b30c818fb1b36d847e7570e99c7e4a889fa8f27eed976e0c906c1533f134a36c46cd6

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-input-dstat
+Dstat input plugin. 

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2016 Yosuke Fujita
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,108 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash). This plugin behave like a [fluent-plugin-dstat](https://github.com/shun0102/fluent-plugin-dstat) on logsash.
+
+
+## install
+
+- Install plugin
+```sh
+bin/logstash-plugin install logstash-input-dstat
+```
+
+## Configuration
+
+```
+input {
+  dstat {
+    option => "-c"
+    interval => 30
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["localhost:9200"]
+  }
+}
+```
+
+### parameters
+
+* option
+    * option for dstat
+    * default: empty
+* interval
+    * interval for executing dstat (seconds)
+    * default: 30
+* stat_hash
+    * set custom stat mapping
+    * ex.) stat_hash => {'total cpu usage' => {'usr' => 'hogehoge' 'sys' => 'fugafuga'}}
+
+### default stat mapping
+
+``` ruby
+{
+  'load avg' => {'1m' => 'loadavg-short', '5m' => 'loadavg-middle', '15m' => 'loadavg-long'},
+  'total cpu usage' => {'usr' => 'cpu-usr', 'sys' => 'cpu-sys', 'idl' => 'cpu-idl', 'wai' => 'cpu-wai', 'hiq' => 'cpu-hiq', 'siq' => 'cpu-siq'},
+  'net/total' => {'recv' => 'net-recv', 'send' => 'net-send'},
+  '/' => {'used' => 'disk-used', 'free' => 'disk-free'},
+  'memory usage' => {'used' => 'mem-used', 'buff' => 'mem-buff', 'cach' => 'mem-cach', 'free' => 'mem-free'},
+  'dsk/total' => {'read' => 'dsk-read', 'writ' => 'dsk-writ'},
+  'paging' => {'in' => 'paging-in', 'out' => 'paging-out'},
+  'system' => {'int' => 'sys-int', 'csw' => 'sys-csw'},
+  'swap' => {'used' => 'swap-used', 'free' => 'swap-free'},
+  'procs' => {'run' => 'procs-run', 'blk' => 'procs-blk', 'new' => 'procs-new'}
+}
+```
+
+### Output
+
+Output example of when using logstash-output-elasticsearch.
+
+
+``` json
+curl -XGET localhost:9200/logstash-*/_search
+
+{
+  "took": 32,
+  "timed_out": false,
+  "_shards": {
+    "total": 3,
+    "successful": 3,
+    "failed": 0
+  },
+  "hits": {
+    "total": 2,
+    "max_score": 3.2686834,
+    "hits": [
+      {
+        "_index": "logstash-2016.07.06",
+        "_type": "logs",
+        "_id": "AVW-_erHoa7OXPDibi4W",
+        "_score": 3.2686834,
+        "_source": {
+          "stat": "cpu-usr",
+          "value": "73.810",
+          "host": "localhost.localdomain",
+          "@version": "1",
+          "@timestamp": "2016-07-06T06:55:58.890Z"
+        }
+      },
+      {
+        "_index": "logstash-2016.07.06",
+        "_type": "logs",
+        "_id": "AVW-_hX2oa7OXPDibi4c",
+        "_score": 3.2686834,
+        "_source": {
+          "stat": "cpu-usr",
+          "value": "0.0",
+          "host": "localhost.localdomain",
+          "@version": "1",
+          "@timestamp": "2016-07-06T06:56:09.950Z"
+        }
+      }
+    ]
+  }
+}
+```

data/lib/logstash/inputs/dstat.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+require "socket"
+require 'csv'
+
+class LogStash::Inputs::Dstat < LogStash::Inputs::Base
+  config_name "dstat"
+
+  default :codec, "plain"
+
+  config :option, :validate => :string, :default => ""
+
+  config :interval, :validate => :number, :default => 30
+
+  config :tmpfile, :validate => :string, :default => "/tmp/logstash-dstat.csv"
+
+  config :stat_hash, :validate => :hash, :default => {
+    'load avg' => {'1m' => 'loadavg-short', '5m' => 'loadavg-middle', '15m' => 'loadavg-long'},
+    'total cpu usage' => {'usr' => 'cpu-usr', 'sys' => 'cpu-sys', 'idl' => 'cpu-idl', 'wai' => 'cpu-wai', 'hiq' => 'cpu-hiq', 'siq' => 'cpu-siq'},
+    'net/total' => {'recv' => 'net-recv', 'send' => 'net-send'},
+    '/' => {'used' => 'disk-used', 'free' => 'disk-free'},
+    'memory usage' => {'used' => 'mem-used', 'buff' => 'mem-buff', 'cach' => 'mem-cach', 'free' => 'mem-free'},
+    'dsk/total' => {'read' => 'dsk-read', 'writ' => 'dsk-writ'},
+    'paging' => {'in' => 'paging-in', 'out' => 'paging-out'},
+    'system' => {'int' => 'sys-int', 'csw' => 'sys-csw'},
+    'swap' => {'used' => 'swap-used', 'free' => 'swap-free'},
+    'procs' => {'run' => 'procs-run', 'blk' => 'procs-blk', 'new' => 'procs-new'}
+  }
+
+  public
+  def register
+    @logger.info("Registering Dstat Input", :type => @type, :command => @option, :interval => @interval)
+    @host = Socket.gethostname
+    @command = 'dstat ' + option + ' --output ' + @tmpfile + ' 1 1'
+  end
+
+  def run(queue)
+    while !stop?
+      clear_tmpfile(@tmpfile)
+      lines = exec_dstat(@command, @tmpfile)
+      events = create_dstat_events(lines)
+      events.each{|event|
+        decorate(event)
+        queue << event
+      }
+      Stud.stoppable_sleep(@interval) { stop? }
+    end
+  end
+
+  def stop
+  end
+
+  def exec_dstat(cmd, tmpfile)
+    @logger.debug? && @logger.debug("Executing dstat", :command => cmd)
+    begin
+      `#{cmd}`
+      File.open(tmpfile) do |file|
+        file.read.split("\n")
+      end
+    rescue Exception => e
+      @logger.error("Exception while running dstat",
+        :command => option, :e => e, :backtrace => e.backtrace)
+    ensure
+      stop
+    end
+  end
+
+  def clear_tmpfile(file)
+    File.open(file,"w") do |file|
+    end
+  end
+
+  def create_dstat_events(lines)
+    events = []
+    top_columns = []
+    second_columns = []
+
+    lines.each_with_index do |line, line_number|
+      line.delete!("\"")
+      next if line == ""
+      case line_number
+      when 0..4
+      when 5
+        top_columns = CSV.parse_line(line)
+        top_columns.each_with_index do |value, i|
+          if value.nil? || value == ""
+            top_columns[i] = top_columns[i-1]
+          end
+        end
+      when 6
+        second_columns = CSV.parse_line(line)
+      when 7
+      when 8
+        CSV.parse_line(line).each_with_index do |value, i|
+          stat = resolve_stat(top_columns[i], second_columns[i])
+          if !stat.nil?
+            event = LogStash::Event.new("stat" => stat, "value" => value, "host" => @host)
+            events << event
+          end
+        end
+      end
+    end
+
+    events
+  end
+
+  def resolve_stat(top_column, second_column)
+    @stat_hash[top_column] ? @stat_hash[top_column][second_column] : nil
+  end
+end

data/logstash-input-dstat.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-dstat'
+  s.version         = '2.0.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "This example input streams a string at a definable interval."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["yfujita"]
+  s.email = 'yfujita@n2sm.net'
+  s.homepage = ""
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/inputs/dstat_spec.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/dstat"
+
+describe LogStash::Inputs::Dstat do
+
+  it_behaves_like "an interruptible input plugin" do
+    let(:config) { { "interval" => 100 } }
+  end
+
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-dstat
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+platform: ruby
+authors:
+- yfujita
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+email: yfujita@n2sm.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/dstat.rb
+- logstash-input-dstat.gemspec
+- spec/inputs/dstat_spec.rb
+homepage: ''
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
+summary: This example input streams a string at a definable interval.
+test_files:
+- spec/inputs/dstat_spec.rb