fluent-plugin-zabbix-simple-bufferd 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MWIzOWRlNDRmYWZiZjNhMDk1MGFiYjUyNzVkM2EzOTYwNmExOWE1MA==
+  data.tar.gz: !binary |-
+    MGYyZWY1NjM0ZGUxNWM3OGRlMTg5MzBhN2FiOGU1YzU1MDJjMTc3Yg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZWQ5MWFmMTUyMDBlMDc2YzhkMWVjYmE0NzNjZmQyMzMzOWFlNDkzZDY2OTM5
+    YzllYmVkZmE1ODMzNTI0MDcwYzM2ZWY4ZTdlMDE4ZjBhMjFjZGM2NGU1NWI1
+    NDcwYjE4Y2NmNzI1NWVlMjMyYjgzYTA3MmFmYmFhODgxNjQ5MWM=
+  data.tar.gz: !binary |-
+    MmViNjcyMTFlODgxM2Y1MWMyMmE3ZTU3OGM5OGEyMmQ2ZWUwMjAyMTA4YTkw
+    OWRjNzlmZjRiNDJiNGRjZTdiNWRmOTUzNThkNmM3NWU2YzhhZDAwMTNkNTY4
+    MGM4ODEyNTUzYWE1YTg5ZjNjYTEwY2ZlODdkYTVmYTA0OTc4MmI=

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-zabbix-simple.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-zabbix-simple (1.0.0)
+      fluentd
+      zabbix
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.1.0)
+      iobuffer (>= 1.0.0)
+    fluentd (0.10.31)
+      cool.io (~> 1.1.0)
+      http_parser.rb (~> 0.5.1)
+      json (>= 1.4.3)
+      msgpack (~> 0.4.4)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.5.3)
+    iobuffer (1.1.2)
+    json (1.7.7)
+    msgpack (0.4.7)
+    yajl-ruby (1.1.0)
+    zabbix (0.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-zabbix-simple!
+  fluentd

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2013 NAKANO Hideo
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README-ja.md

@@ -0,0 +1,8 @@
+# fluent-plugin-zabbix-simple-bufferd
+
+## 概要
+
+fluent-plugin-zabbix-simpleが継承してるクラスを
+
+Fluent::Output -> Fluent::BufferedOutputに変更しただけです。
+若干クラス名等は変えましたが、基本それだけです。

data/README.md

@@ -0,0 +1,136 @@
+# fluent-plugin-zabbix-simple
+
+## What is this
+
+**fluent-plugin-zabbix-simple** is a [fluentd](http://fluentd.org/ "fluentd") output plugin to send values to [Zabbix](http://www.zabbix.com/ "Zabbix") Server.
+
+fluentd collect logs as JSON.
+fluent-plugin-zabbix-simple converts **fluentd's JSON key** to **Zabbix key**, and sends _Zabbix key_ and its value to Zabbix Server.
+
+* You can define multiple key which you want to send, no size limitation.
+
+* You can use regex both **key-pattern** and **key-replacement**.
+
+  * **key-pattern** (simply called **pattern**) is a key that is matched against _fluentd's JSON key_.
+
+  * **key-replacement** (simple called **replacement**) is a key that is send to Zabbix Server if _pattern_ matches _fluentd's JSON key_. Before sending to Zabbix Server, _fluentd's JSON key_ is converted to _Zabbix key_ according to the replacement.
+
+## How to Install
+
+execute command `gem install`:
+
+    $ sudo gem install fluent-plugin-zabbix-simple
+
+[td-agent](http://docs.fluentd.org/articles/install-by-rpm#what-is-td-agent) has its own Ruby ecosystem.
+If you have installed td-agent, you would use `gem` command included with td-agent.
+
+    $ sudo /usr/lib64/fluent/ruby/bin/gem isntall fluent-plugin-zabbix-simple
+
+## Installation Check
+
+### Zabbix Server Configuration
+
+In advance, You shoud define zabbix items like this:
+
+    Key: httpd.status[2xx]
+    Type: Zabbix trapper
+    Type of information: Numeric(unsigned)
+
+    Key: httpd.status[3xx]
+    Type: Zabbix trapper
+    Type of information: Numeric(unsigned)
+
+Tips:
+
+* You must set `Type` to `Zabbix trapper`.
+
+* You must choose a appropriate `Type of information`.
+
+### Plugin Configuration and Test
+
+To test fluent-plugin-zabbix-simple, create file:
+
+    <source>
+      type forward
+    </source>
+    <match httpd.access.status_count>
+      type zabbix_simple
+      zabbix_server 192.168.0.1
+      map_key1 httpd.access_(...)_count httpd.status[\1]
+    </match>
+
+Save above file to `fluentd.conf`, execute `fluentd`:
+
+    $ fluentd -c ./fluent.conf -vv
+
+If you have installed td-agent, execute a command like this:
+
+    $ /usr/sbin/td-agent -c ./fluent.conf -vv
+
+Open another termina, send a test message to fluentd server.
+
+    $ echo '{"httpd.access_2xx_count":321}' | fluent-cat httpd.access.status_count
+
+after a few seconds, confirm that the 321 has been recorded in the Zabbix Server.
+
+
+## Configuration
+
+name | type | description
+-----|------|------
+type | string | type of plugin. fluent-plugin-zabbix-simple has "zabbix_simple".
+zabbix_server | string | IP address or hostname of Zabbix Server.
+port | integer | port no which zabbix Server uses(default is 10051).
+host | string | hostname of sender(default is `Socket.gethostname`).
+key_size | integer | size of map_key(default is 20)
+map_key[n] | string | a space separated _pattern_ and _replacement. You can use `map_key0` as 0th map_key.
+
+
+## Use Many Keys
+
+By default, key_map is scanned up to 20.
+You must specify `key_size` if you want to use key_map more than 20.
+
+    <match httpd.access.status_count>
+      type zabbix_simple
+      zabbix_server 192.168.0.1
+      key_size 25
+      map_key1  pattern1  replace1
+      map_key2  pattern2  replace2
+      map_key3  pattern3  replace3
+      map_key4  pattern4  replace4
+      map_key5  pattern5  replace5
+      map_key6  pattern6  replace6
+      map_key7  pattern7  replace7
+      map_key8  pattern8  replace8
+      map_key9  pattern9  replace9
+      map_key10 pattern10 replace10
+      map_key11 pattern11 replace11
+      map_key12 pattern12 replace12
+      map_key13 pattern13 replace13
+      map_key14 pattern14 replace14
+      map_key15 pattern15 replace15
+      map_key16 pattern16 replace16
+      map_key17 pattern17 replace17
+      map_key18 pattern18 replace18
+      map_key19 pattern19 replace19
+      map_key20 pattern20 replace20
+      map_key21 pattern21 replace21
+      map_key22 pattern22 replace22
+      map_key23 pattern23 replace23
+      map_key24 pattern24 replace24
+      map_key25 pattern25 replace25
+    </match>
+
+## Detail of matching
+
+fluent-plugin-zabbix-simple lookups from `map_key0` up to map_key[key_size], use the first matched map_key, ignore the rest of map_keys.
+
+## Legal Notification
+
+### Copyright
+Copyright (c) 2013 NAKANO Hideo
+
+### License
+Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+# -*- mode:ruby -*-
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-zabbix-simple-bufferd.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+# -*- mode:ruby -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Katsuhiro Miura"]
+  gem.email         = ["MiuraKatsu@gmail.com"]
+  gem.description   = %q{Output data plugin to Zabbix}
+  gem.summary       = %q{Output data plugin to Zabbix (like zabbix_sender)}
+  gem.homepage      = "https://github.com/katsuhiro-miura/fluent-plugin-zabbix-simple"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-zabbix-simple-bufferd"
+  gem.require_paths = ["lib"]
+  gem.version       = "1.0.0"
+
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "zabbix"
+end

data/lib/fluent/plugin/out_zabbix_simple_bufferd.rb

@@ -0,0 +1,119 @@
+#
+# Fluent::ZabbixSimpleOutput
+#
+# Copyright (C) 2013 NAKANO Hideo
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+class Fluent::ZabbixSimpleBufferdOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('zabbix_simple_bufferd', self)
+
+  def initialize
+    super
+    require 'zabbix'
+    require 'socket'
+  end
+
+  attr_reader :zabbix_server, :port, :host, :key_size, :map_keys
+
+  config_param :zabbix_server, :string
+  config_param :port, :integer,            :default => 10051
+  config_param :host, :string,             :default => Socket.gethostname
+  config_param :key_size, :integer,        :default => 20
+
+  KeyMap = Struct.new(:id, :pattern, :replace)
+
+  def configure(conf)
+    super
+
+    if @zabbix_server.nil?
+      raise Fluent::ConfigError, "missing zabbix_server"
+    end
+
+    @map_keys = []
+    (0..@key_size).each do |i|
+      next unless conf["map_key#{i}"]
+      pattern,replace = conf["map_key#{i}"].split(' ', 2)
+      @map_keys.push(KeyMap.new(i, Regexp.new(pattern), replace))
+    end
+
+    if @map_keys.nil? or @map_keys.size == 0
+      raise Fluent::ConfigError, "missing map_key[0..]"
+    end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def create_zbx_sender
+    Zabbix::Sender.new(:host => @zabbix_server, :port => @port)
+  end
+
+  def send(zbx_sender, name, value, time)
+    begin
+      $log.debug { "name: #{name}, value: #{value}, time: #{time}" }
+
+      opts = { :host => @host, :ts => time }
+      status = zbx_sender.send_data(name, value.to_s, opts)
+
+    rescue IOError, EOFError, SystemCallError
+      # server didn't respond
+      $log.warn "Zabbix::Sender.send_data raises exception: #{$!.class}, '#{$!.message}'"
+      status = false
+    end
+    unless status
+      $log.warn "failed to send to zabbix_server `#{@zabbix_server}(port:`#{@port}`), host:#{@host} '#{name}': #{value}"
+    end
+  end
+
+  def emit(tag, es, chain)
+    zbx_sender = nil
+    begin
+      $log.trace { "connecting to zabbix server `#{@zabbix_server}(port:`#{@port}`)" }
+      zbx_sender = create_zbx_sender
+      zbx_sender.connect
+      $log.trace "done connected to zabbix server"
+    rescue
+      $log.warn "could not connect to zabbix server `#{@zabbix_server}(port:`#{@port})`, exception: #{$!.class}, '#{$!.message}'"
+    end
+
+    if zbx_sender
+      es.each do |time, record|
+        record.each do |key,value|
+          @map_keys.each do |map|
+            zbx_key = map_key(key, map.pattern, map.replace)
+            next unless zbx_key
+            send(zbx_sender, zbx_key, value, time)
+          end
+        end
+      end
+      zbx_sender.disconnect
+    end
+
+    # call next chain
+    chain.next
+  end
+
+  def map_key(key, pattern, replace)
+    unless pattern =~ key
+      nil
+    else
+      key.sub(pattern, replace)
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_zabbix_simple'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_zabbix_simple.rb

@@ -0,0 +1,186 @@
+require 'helper'
+
+module Mock
+  class ::Fluent::ZabbixSimpleOutput
+    def self.mock!
+      include MockZabbixSimpleOutput
+      alias_method :old_create_zbx_sender, :create_zbx_sender 
+      alias_method :create_zbx_sender, :mock_create_zbx_sender
+    end
+    def self.unmock!
+      return unless method_defined? :mock_say
+      alias_method :say, :old_say
+    end
+  end
+
+  class MockZbxSender
+    @@data = []
+    def send_data(key, value, opts)
+      @@data.push([key, value, opts])
+    end
+    def connect
+    end
+    def disconnect
+    end
+    def configured?()
+      true
+    end
+    def self.data
+      @@data
+    end
+    def self.clearData
+      @@data = []
+    end
+  end
+
+  module MockZabbixSimpleOutput
+    def mock_create_zbx_sender
+      MockZbxSender.new
+    end
+  end
+end
+
+class ZabbixOutputSimpleTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    zabbix_server  127.0.0.1
+    host           clienthost
+    map_key1        x1 y1
+    map_key2        x2 y2
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::ZabbixSimpleOutput, tag).configure(conf)
+  end
+
+  def test_emit001
+    ::Mock::MockZbxSender.clearData
+    Fluent::ZabbixSimpleOutput.mock!
+    d = create_driver
+    d.emit({"x1" => "test value of x1"})
+    Fluent::ZabbixSimpleOutput.unmock!
+
+    assert_equal(1, ::Mock::MockZbxSender.data.size)
+    assert_equal("y1", ::Mock::MockZbxSender.data[0][0])
+    assert_equal("test value of x1", ::Mock::MockZbxSender.data[0][1])
+    assert_equal("clienthost", ::Mock::MockZbxSender.data[0][2][:host])
+    assert_block do
+      ::Mock::MockZbxSender.data[0][2][:ts] > 0
+    end
+    ::Mock::MockZbxSender.clearData
+  end
+
+  def test_emit002
+    ::Mock::MockZbxSender.clearData
+
+    Fluent::ZabbixSimpleOutput.mock!
+    d = create_driver
+    d.emit({"x3" => "test value of x3"})
+    Fluent::ZabbixSimpleOutput.unmock!
+
+    assert_equal(0, ::Mock::MockZbxSender.data.size)
+
+    ::Mock::MockZbxSender.clearData
+  end
+
+  def test_emit003
+    ::Mock::MockZbxSender.clearData
+
+    Fluent::ZabbixSimpleOutput.mock!
+    d = create_driver %{
+      zabbix_server  127.0.0.1
+      map_key1       input_([^_]+)_count httpd.count[\\1]
+    }
+    d.emit({"input_unmatched_count" => 10, "input_unmatched_rate" => 0.431, "input_unmatched_percentage" => 0.73})
+    d.emit({"input_status2xx_count" => 1884540035, "input_status2xx_rate" => 2041995578.0 / 730602602.0, "input_status2xx_percentage" => 422483907.0 / 426370718.0})
+    Fluent::ZabbixSimpleOutput.unmock!
+
+    assert_equal(2, ::Mock::MockZbxSender.data.size)
+    assert_equal("httpd.count[unmatched]", ::Mock::MockZbxSender.data[0][0])
+    assert_equal("10", ::Mock::MockZbxSender.data[0][1])
+    assert_equal("httpd.count[status2xx]", ::Mock::MockZbxSender.data[1][0])
+    assert_equal("1884540035", ::Mock::MockZbxSender.data[1][1])
+
+    ::Mock::MockZbxSender.clearData
+  end
+
+  def test_configure001
+    d = create_driver
+    assert_equal("127.0.0.1", d.instance.zabbix_server)
+    assert_equal(10051, d.instance.port)
+    assert_equal("clienthost", d.instance.host)
+    assert_equal(20, d.instance.key_size)
+    assert_equal(2, d.instance.map_keys.size)
+
+    assert_equal(1, d.instance.map_keys[0][0])
+    assert_equal("/x1/", d.instance.map_keys[0][1].inspect)
+    assert_equal("y1", d.instance.map_keys[0][2])
+    assert_equal(2, d.instance.map_keys[1][0])
+    assert_equal("/x2/", d.instance.map_keys[1][1].inspect)
+    assert_equal("y2", d.instance.map_keys[1][2])
+
+    assert_equal(1, d.instance.map_keys[0].id)
+    assert_equal("/x1/", d.instance.map_keys[0].pattern.inspect)
+    assert_equal("y1", d.instance.map_keys[0].replace)
+    assert_equal(2, d.instance.map_keys[1].id)
+    assert_equal("/x2/", d.instance.map_keys[1].pattern.inspect)
+    assert_equal("y2", d.instance.map_keys[1].replace)
+  end
+
+  def test_configure002
+    assert_raise Fluent::ConfigError do
+      d = create_driver %[
+        host clienthost
+      ]
+    end
+  end
+
+  def test_configure003
+    assert_raise Fluent::ConfigError do
+      d = create_driver %[
+        zabbix_server  127.0.0.1
+        host clienthost
+      ]
+    end
+  end
+
+  def test_configure004
+    d = create_driver %[
+        zabbix_server  127.0.0.1
+        port 490478390,
+        host clienthost
+        map_key3 x1 y1
+    ]
+
+    assert_equal("127.0.0.1", d.instance.zabbix_server)
+    assert_equal(490478390, d.instance.port)
+    assert_equal("clienthost", d.instance.host)
+    assert_equal(20, d.instance.key_size)
+    assert_equal(1, d.instance.map_keys.size)
+    assert_equal(3, d.instance.map_keys[0][0])
+    assert_equal("/x1/", d.instance.map_keys[0][1].inspect)
+    assert_equal("y1", d.instance.map_keys[0][2])
+    assert_equal(3, d.instance.map_keys[0].id)
+    assert_equal("/x1/", d.instance.map_keys[0].pattern.inspect)
+    assert_equal("y1", d.instance.map_keys[0].replace)
+  end
+
+  def test_map_key
+    d = create_driver
+
+    assert_equal("http.statusCount[2xx]", 
+      d.instance.map_key("input_status2xx_count", /^.+_status(...)_count$/,
+        'http.statusCount[\1]'))
+    assert_equal("http.statusCount[2xx]", 
+      d.instance.map_key("input_status2xx_count",
+         Regexp.new('^.+_status(...)_count$'),
+        'http.statusCount[\1]'))
+    assert_nil( 
+      d.instance.map_key("input_status2xx_rate",
+         Regexp.new('^.+_status(...)_count$'),
+        'http.statusCount[\1]'))
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-zabbix-simple-bufferd
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Katsuhiro Miura
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zabbix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Output data plugin to Zabbix
+email:
+- MiuraKatsu@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README-ja.md
+- README.md
+- Rakefile
+- fluent-plugin-zabbix-simple-bufferd.gemspec
+- lib/fluent/plugin/out_zabbix_simple_bufferd.rb
+- test/helper.rb
+- test/plugin/test_out_zabbix_simple.rb
+homepage: https://github.com/katsuhiro-miura/fluent-plugin-zabbix-simple
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.6
+signing_key: 
+specification_version: 4
+summary: Output data plugin to Zabbix (like zabbix_sender)
+test_files:
+- test/helper.rb
+- test/plugin/test_out_zabbix_simple.rb
+has_rdoc: