fluent-plugin-out-http 0.1.0

data/.gitignore

@@ -0,0 +1,7 @@
+*~
+\#*
+.\#*
+*.gem
+.bundle
+Gemfile.lock
+vendor

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out-http.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,23 @@
+# fluent-plugin-out-http
+
+A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint.
+
+## Configs
+
+    <match *>
+      type http
+      endpoint_url    http://localhost.local/api/
+      http_method     put
+      serializer      json
+      rate_limit_msec 100
+      authentication  basic
+      username        alice
+      password        bobpop
+    </match>
+
+----
+
+Heavily based on [fluent-plugin-growthforecast][2]
+
+  [1]: http://fluentd.org/
+  [2]: https://github.com/tagomoris/fluent-plugin-growthforecast

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-out-http.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-out-http"
+  gem.version       = "0.1.0"
+  gem.authors       = ["Marica Odagaki"]
+  gem.email         = ["ento.entotto@gmail.com"]
+  gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint}
+  gem.description   = gem.summary
+  gem.homepage      = "https://github.com/ento/fluent-plugin-out-http"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "json"
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/out_http.rb

@@ -0,0 +1,127 @@
+require 'json'
+
+class Fluent::HTTPOutput < Fluent::Output
+  Fluent::Plugin.register_output('http', self)
+
+  def initialize
+    super
+    require 'net/http'
+    require 'uri'
+  end
+
+  # Endpoint URL ex. localhost.local/api/
+  config_param :endpoint_url, :string
+
+  # HTTP method
+  config_param :http_method, :string, :default => :post
+  
+  # form | json
+  config_param :serializer, :string, :default => :form
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # nil | 'none' | 'basic'
+  config_param :authentication, :string, :default => nil 
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => ''
+
+  def configure(conf)
+    super
+
+    serializers = [:json, :form]
+    @serializer = if serializers.include? @serializer.intern
+                    @serializer.intern
+                  else
+                    :form
+                  end
+
+    http_methods = [:get, :put, :post, :delete]
+    @http_method = if http_methods.include? @http_method.intern
+                    @http_method.intern
+                  else
+                    :post
+                  end
+
+    @auth = case @authentication
+            when 'basic' then :basic
+            else
+              :none
+            end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    @endpoint_url
+  end
+
+  def set_body(req, tag, time, record)
+    if @serializer == :json
+      set_json_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    req
+  end
+
+  def set_json_body(req, data)
+    req.body = JSON.dump(data)
+    req['Content-Type'] = 'application/json'
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    return req, uri
+  end
+
+  def send_request(req, uri)    
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      $log.info('Dropped request due to rate limiting')
+      return
+    end
+    
+    res = nil
+    begin
+      if @auth and @auth == :basic
+        req.basic_auth(@username, @password)
+      end
+      @last_request_time = Time.now.to_f
+      res = Net::HTTP.new(uri.host, uri.port).start {|http| http.request(req) }
+    rescue IOError, EOFError, SystemCallError
+      # server didn't respond
+      $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{$!.class}, '#{$!.message}'"
+    end
+    unless res and res.is_a?(Net::HTTPSuccess)
+      $log.warn "failed to #{req.method} #{uri} (#{res.code} #{res.message} #{res.body})"
+    end
+  end
+
+  def handle_record(tag, time, record)
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri)
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+    chain.next
+  end
+end

data/lib/fluent/test/http_output_test.rb

@@ -0,0 +1,54 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+class Test::Unit::TestCase
+end
+
+require 'webrick'
+
+# to handle POST/PUT/DELETE ...
+module WEBrick::HTTPServlet
+  class ProcHandler < AbstractServlet
+    alias do_POST   do_GET
+    alias do_PUT    do_GET
+    alias do_DELETE do_GET
+  end
+end
+
+def get_code(server, port, path, headers={})
+  require 'net/http' 
+  Net::HTTP.start(server, port){|http|
+    http.get(path, headers).code
+  } 
+end 
+def get_content(server, port, path, headers={})
+  require 'net/http'
+  Net::HTTP.start(server, port){|http|
+    http.get(path, headers).body
+  } 
+end

data/test/plugin/test_out_http.rb

@@ -0,0 +1,281 @@
+require 'json'
+require 'fluent/test/http_output_test'
+require 'fluent/plugin/out_http'
+
+
+TEST_LISTEN_PORT = 5126
+
+
+class HTTPOutputTestBase < Test::Unit::TestCase
+  # setup / teardown for servers
+  def setup
+    Fluent::Test.setup
+    @posts = []
+    @puts = []
+    @prohibited = 0
+    @auth = false
+    @dummy_server_thread = Thread.new do
+      srv = if ENV['VERBOSE']
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
+            else
+              logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
+            end
+      begin
+        allowed_methods = %w(POST PUT)
+        srv.mount_proc('/api/') { |req,res|
+          unless allowed_methods.include? req.request_method
+            res.status = 405
+            res.body = 'request method mismatch'
+            next
+          end
+          if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!'
+            # ok, authorized
+          elsif @auth
+            res.status = 403
+            @prohibited += 1
+            next
+          else
+            # ok, authorization not required
+          end
+
+          record = {:auth => nil}
+          if req.content_type == 'application/json'
+            record[:json] = JSON.parse(req.body)
+          else
+            record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)]
+          end
+
+          instance_variable_get("@#{req.request_method.downcase}s").push(record)
+
+          res.status = 200
+        }
+        srv.mount_proc('/') { |req,res|
+          res.status = 200
+          res.body = 'running'
+        }
+        srv.start
+      ensure
+        srv.shutdown
+      end
+    end
+
+    # to wait completion of dummy server.start()
+    require 'thread'
+    cv = ConditionVariable.new
+    watcher = Thread.new {
+      connected = false
+      while not connected
+        begin
+          get_content('localhost', TEST_LISTEN_PORT, '/')
+          connected = true
+        rescue Errno::ECONNREFUSED
+          sleep 0.1
+        rescue StandardError => e
+          p e
+          sleep 0.1
+        end
+      end
+      cv.signal
+    } 
+    mutex = Mutex.new
+    mutex.synchronize {
+      cv.wait(mutex)
+    } 
+  end
+
+  def test_dummy_server
+    host = '127.0.0.1'
+    port = TEST_LISTEN_PORT
+    client = Net::HTTP.start(host, port)
+
+    assert_equal '200', client.request_get('/').code
+    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '1', @posts[0][:form]['number']
+    assert_equal 'gauge', @posts[0][:form]['mode']
+    assert_nil @posts[0][:auth]
+
+    @auth = true
+
+    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
+
+    req_with_auth = lambda do |number, mode, user, pass|
+      url = URI.parse("http://#{host}:#{port}/api/service/metrics/pos")
+      req = Net::HTTP::Post.new(url.path)
+      req.basic_auth user, pass
+      req.set_form_data({'number'=>number, 'mode'=>mode})
+      req
+    end
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code
+
+    assert_equal 2, @posts.size
+
+  end
+
+  def teardown
+    @dummy_server_thread.kill
+    @dummy_server_thread.join
+  end
+end
+
+class HTTPOutputTest < HTTPOutputTestBase
+  CONFIG = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+  ]
+
+  CONFIG_JSON = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer json
+  ]
+
+  CONFIG_PUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    http_method put
+  ]
+
+  RATE_LIMIT_MSEC = 1200
+
+  CONFIG_RATE_LIMIT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    rate_limit_msec #{RATE_LIMIT_MSEC}
+  ]
+
+  def create_driver(conf=CONFIG, tag='test.metrics')
+    Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :form, d.instance.serializer
+
+    d = create_driver CONFIG_JSON
+    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :json, d.instance.serializer
+  end
+
+  def test_emit_form
+    d = create_driver
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal '50', record[:form]['field1']
+    assert_equal '20', record[:form]['field2']
+    assert_equal '10', record[:form]['field3']
+    assert_equal '1', record[:form]['otherfield']
+    assert_nil record[:auth]
+
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 2, @posts.size
+  end
+
+  def test_emit_form_put
+    d = create_driver CONFIG_PUT
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @puts.size
+    record = @puts[0]
+
+    assert_equal '50', record[:form]['field1']
+    assert_nil record[:auth]
+
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @puts.size
+  end
+
+  def test_emit_json
+    d = create_driver CONFIG_JSON
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal 50, record[:json]['field1']
+    assert_equal 20, record[:json]['field2']
+    assert_equal 10, record[:json]['field3']
+    assert_equal 1, record[:json]['otherfield']
+    assert_nil record[:auth]
+  end
+
+  def test_rate_limiting
+    d = create_driver CONFIG_RATE_LIMIT
+    record = { :k => 1 }
+
+    last_emit = _current_msec
+    d.emit(record)
+    d.run
+
+    assert_equal 1, @posts.size
+
+    d.emit({})
+    d.run
+    assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval"
+    assert_equal 1, @posts.size
+
+    sleep (last_emit + RATE_LIMIT_MSEC - _current_msec) * 0.001
+    
+    assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
+    d.emit(record)
+    d.run
+    assert_equal 2, @posts.size
+  end
+
+  def _current_msec
+    Time.now.to_f * 1000
+  end
+  
+  def test_auth
+    @auth = true # enable authentication of dummy server
+
+    d = create_driver(CONFIG, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @prohibited
+
+    d = create_driver(CONFIG + %[
+      authentication basic
+      username alice
+      password wrong_password
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @prohibited
+
+    d = create_driver(CONFIG + %[
+      authentication basic
+      username alice
+      password secret!
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 1, @posts.size
+    assert_equal 2, @prohibited
+  end
+
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-out-http
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Marica Odagaki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-14 00:00:00.000000000 +09:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &70205234747520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70205234747520
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &70205234747080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70205234747080
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: &70205234746660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70205234746660
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: &70205234746240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70205234746240
+description: A generic Fluentd output plugin to send logs to an HTTP endpoint
+email:
+- ento.entotto@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-out-http.gemspec
+- lib/fluent/plugin/out_http.rb
+- lib/fluent/test/http_output_test.rb
+- test/plugin/test_out_http.rb
+has_rdoc: true
+homepage: https://github.com/ento/fluent-plugin-out-http
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: A generic Fluentd output plugin to send logs to an HTTP endpoint
+test_files:
+- test/plugin/test_out_http.rb