fluent-plugin-http-status 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-http-response.gemspec
+gemspec
+gem 'mocha'
+gem 'webmock'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 sugimoto hiroshi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+# Fluent::Plugin::Http::Status
+
+Fluentd input plugin for to get the http status.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-http-status'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-http-status
+
+## Usage
+
+Config
+
+    <source>
+      type http_status
+      tag http.status
+      url https://rubygems.org
+      polling_time 1m
+
+      #=== options ===
+      #port 80                    
+      #basic_user user            
+      #basic_password pass        
+      #proxy_address proxy.test.jp
+      #proxy_port 8080            
+      #proxy_user user            
+      #proxy_password pass        
+      #params server,date         
+      #open_timeout 10            
+      #read_timeout 20            
+    </source>
+    
+    <match http.status>
+      type stdout
+    </match>
+
+Result
+
+    Time : 2012-11-19 03:29:29 +0900
+    Tag : http.status
+    Record : {"url":"https://rubygems.org","host":"rubygems.org","port":443,"request_uri":"/","code":200,"message":"OK","class_name":"Net::HTTPOK","ipaddress":"204.232.149.25","headers":{"server":"nginx/1.2.2", ...
+
+## Copyright
+Copyright (c) 2012 hiro-su
+Apache Licence, Version 2.0

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = false               
+end

data/fluent-plugin-http-status.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-http-status"
+  gem.version       = "0.0.1"
+  gem.authors       = ["hiro-su"]
+  gem.email         = ["h.sugipon@gmail.com"]
+  gem.description   = %q{Fluentd input plugin for to get the http status}
+  gem.summary       = %q{Fluentd input plugin for to get the http status}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "polling"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "polling"
+end

data/http.conf.sample

@@ -0,0 +1,21 @@
+<source>
+  type http_status
+  tag http.status
+  url https://rubygems.org
+  polling_time 1m
+
+  #port 80
+  #basic_user user
+  #basic_password pass
+  #proxy_address proxy.test.jp
+  #proxy_port 8080
+  #proxy_user user
+  #proxy_password pass
+  #params server,date
+  #open_timeout 10
+  #read_timeout 20
+</source>
+
+<match http.*>
+  type stdout
+</match>

data/lib/fluent/plugin/in_http_status.rb

@@ -0,0 +1,139 @@
+require "polling"
+require "net/http"
+require "net/https"
+require "uri" 
+require "socket"
+
+module Fluent
+  class HttpStatusInput < Input
+    Plugin.register_input('http_status',self)
+
+    config_param :tag, :string
+    config_param :url, :string
+    config_param :port, :integer, :default => nil
+    config_param :proxy_address, :string, :default => nil
+    config_param :proxy_port, :integer, :default => nil
+    config_param :proxy_user, :string, :default => nil
+    config_param :proxy_password, :string, :default => nil
+    config_param :open_timeout, :integer, :default => 10
+    config_param :read_timeout, :integer, :default => 20
+    config_param :params, :string, :default => nil
+    config_param :polling_time, :string
+    config_param :basic_user, :string, :default => nil
+    config_param :basic_password, :string, :default => nil
+
+    def configure(conf)
+      super
+      @params = @params.split(',').map{|str| str.strip} unless @params.nil?
+      @polling_time = @polling_time.split(',').map{|str| str.strip} unless @polling_time.nil?
+      raise ConfigError, "snmp: 'polling_time' parameter is required on snmp input" if !@polling_time.nil? && @polling_time.empty?
+      @retry_count = 0
+    end
+
+    def starter
+      Net::HTTP.version_1_2
+      @starter=Thread.new{yield}
+    end
+
+    def start
+      starter{@thread=Thread.new(&method(:run))}
+    end
+
+    def run
+      Polling::run(@polling_time) do
+        record = Hash.new
+        args = {
+          :url => @url,
+          :port => @port,
+          :proxy_address => @proxy_address,
+          :proxy_port => @proxy_port,
+          :proxy_user => @proxy_user,
+          :proxy_password => @proxy_password,
+          :params => @params
+        }
+        Engine.emit(@tag, Engine.now, get_status(record,args))
+        break if @end_flag
+      end
+    rescue TypeError => ex
+      $log.error "run TypeError", :error=>ex.message
+      exit
+    rescue => ex
+      $log.error "run failed", :error=>ex.message
+      sleep(10)
+      @retry_count += 1
+      retry if @retry_count < 30
+    end
+
+    def shutdown
+      @end_flag ||= true
+      @thread.run
+      @thread.join
+      @starter.join
+    end
+
+    private
+
+    def get_status(hash,*args)
+      args.each do |arg|
+        @url = arg[:url]
+        @port = arg[:port]
+        @proxy_address = arg[:proxy_address]
+        @proxy_port = arg[:proxy_port]
+        @proxy_user = arg[:proxy_user]
+        @proxy_password = arg[:proxy_password]
+        @params = arg[:params]
+      end
+
+      start = Time.now
+
+      uri = URI.parse("#{@url}:#{@port}")
+      host = uri.host
+      port = uri.port
+      request_uri = uri.request_uri
+
+      http = Net::HTTP.new(host,port,@proxy_address,@proxy_port,@proxy_user,@proxy_password)
+      http.open_timeout = @open_timeout
+      http.read_timeout = @read_timeout
+
+      if uri.port == 443
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        http.verify_depth = 5
+      end
+
+      req = Net::HTTP::Head.new(request_uri)
+      req.basic_auth @basic_user, @basic_password
+      response = http.request(req)
+
+      hash[:url] = url
+      hash[:host] = host
+      hash[:port] = port
+      hash[:request_uri] = request_uri
+      hash[:proxy_address] = @proxy_address if @proxy_address
+      hash[:proxy_port] = @proxy_port if @proxy_port
+      hash[:code] = response.code.to_i
+      hash[:message] = response.message
+      hash[:class_name] = response.class.name
+      hostent = Socket.gethostbyname(host)
+      hash[:ipaddress] = hostent[3].unpack("C4").join('.')
+      hash[:headers] = Hash.new
+
+      response.each_key{|name|
+        if @params
+          @params.each{|param| hash[:headers][param] = response[param] =~ /^-?\d+$/ ? response[param].to_i : response[param]}
+        else
+          hash[:headers][name] = response[name] =~ /^-?\d+$/ ? response[name].to_i : response[name]
+        end
+      }
+
+      hash[:response_time] = Time.now - start
+
+      return hash
+    rescue Timeout::Error => ex
+      return "Timeout Error : #{ex}"
+    rescue => ex
+      return ex.message
+    end
+
+  end
+end

data/test/helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'test/unit'
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_http_status'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_http_status.rb

@@ -0,0 +1,105 @@
+require 'helper'
+require 'mocha/setup'
+require 'webmock/test_unit'
+require 'socket'
+
+class HttpStatusInputTest < Test::Unit::TestCase
+
+  def setup
+    Fluent::Test.setup
+    @obj = Fluent::HttpStatusInput.new
+    @hash = Hash.new
+  end
+
+  CONFIG = %[
+    tag http.status
+    url http://www.test.ad.jp
+    port 80
+    basic_user user
+    basic_password password
+    proxy_address proxy.test.jp
+    proxy_port 8080
+    proxy_user proxy_user
+    proxy_password proxy_password
+    polling_time 10s
+    #params server,date
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::HttpStatusInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal "http.status", d.instance.tag
+    assert_equal "http://www.test.ad.jp", d.instance.url
+    assert_equal 80, d.instance.port
+    assert_equal "user", d.instance.basic_user
+    assert_equal "password", d.instance.basic_password
+    assert_equal "proxy.test.jp", d.instance.proxy_address
+    assert_equal "proxy_user", d.instance.proxy_user
+    assert_equal "proxy_password", d.instance.proxy_password
+    assert_equal 8080, d.instance.proxy_port
+    assert_equal ["10s"], d.instance.polling_time
+    #assert_equal ["server","date"], d.instance.params
+  end
+
+  def test_get_status
+    d = create_driver
+    args = {
+      :url => d.instance.url,
+      :port => d.instance.port,
+      :proxy_address => d.instance.proxy_address,
+      :proxy_port => d.instance.proxy_port,
+      :proxy_user => d.instance.proxy_user,
+      :proxy_password => d.instance.proxy_password,
+      :params => d.instance.params
+    }
+
+    headers = {}
+    headers[:headers] = {
+      "server" => "Apache",
+      "date" => "Sun, 18 Nov 2012 17:12:09 GMT",
+      "last-modified" => "Thu, 15 Nov 2012 02:00:30 GMT",
+      "etag" => "\"68c9e3-4356-3fe2b80\"",
+      "accept-ranges" => "bytes",
+      "content-length" => "17238",
+      "content-type" => "text/html",
+      "via" => "1.1 www.test.ad.jp",
+      "connection" => "close"
+    }
+
+    Socket.stubs(:gethostbyname).returns(["www.test.ad.jp", [], 0, "\xC0\xA8\x00\x01"])
+    WebMock.stub_request(:head, "www.test.ad.jp").to_return(:status => 200, :headers => headers[:headers])
+    res_data = @obj.__send__(:get_status,@hash,args)
+    res_data[:response_time] = 0.001239
+    
+    result_hash = {
+      :url=>"http://www.test.ad.jp",
+      :host=>"www.test.ad.jp", 
+      :port=>80, :request_uri=>"/",
+      :proxy_address=>"proxy.test.jp",
+      :proxy_port=>8080,
+      :code=>200,
+      :message=>"",
+      :class_name=>"Net::HTTPOK",
+      :ipaddress=>"192.168.0.1",
+      :response_time=>0.001239,
+      :headers=>{
+        "server"=>"Apache",
+        "date"=>"Sun, 18 Nov 2012 17:12:09 GMT",
+        "last-modified"=>"Thu, 15 Nov 2012 02:00:30 GMT", 
+        "etag"=>"\"68c9e3-4356-3fe2b80\"",
+        "accept-ranges"=>"bytes",
+        "content-length"=>17238,
+        "content-type"=>"text/html",
+        "via"=>"1.1 www.test.ad.jp",
+        "connection"=>"close"
+      }
+    }
+
+    assert_equal result_hash, res_data
+  end
+
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http-status
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- hiro-su
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: polling
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: polling
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd input plugin for to get the http status
+email:
+- h.sugipon@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-http-status.gemspec
+- http.conf.sample
+- lib/fluent/plugin/in_http_status.rb
+- test/helper.rb
+- test/plugin/test_in_http_status.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Fluentd input plugin for to get the http status
+test_files:
+- test/helper.rb
+- test/plugin/test_in_http_status.rb