fluent-plugin-out-falcon 0.1.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 93262b579cc60c929f865d81171a46b4d610738d
+  data.tar.gz: 6d0e4885faaebce8db8a1ca12488f9f0fd426a3c
+SHA512:
+  metadata.gz: 00d5bc6bb7e9ed15148858bc792a3aed8290cea197e7f66c8d22cfe2579667583df53e79edcbd8a3b2c7ff3b7e2e026b184ddc48f063703c8d8e51c2b1764aef
+  data.tar.gz: 6d8d6d3b27bc2cdbca11cd8238d127a90ecc5b14337bad1360d479e0e5bb5abb8d7bd62549b3b3a0610ce1fed8a8ec217193de2cd39bbfdc2bb811c52f664d27

data/.gitignore

@@ -0,0 +1,8 @@
+*~
+\#*
+.\#*
+*.gem
+.bundle
+.ruby-version
+Gemfile.lock
+vendor

data/.travis.yml

@@ -0,0 +1,21 @@
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - 2.2
+  - ruby-head
+  - rbx-2
+
+os:
+  - linux
+  - osx
+
+gemfile:
+  - Gemfile
+
+script: bundle exec rake test
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx-2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out-falcon.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,24 @@
+# fluent-plugin-out-falcon, a plugin for [Fluentd](http://fluentd.org)
+
+A [fluentd][1] output plugin for sending logs to [falcon][2]'s push api.
+
+## Configuration options
+
+    <match *>
+      type falcon
+      endpoint_url    http://localhost.local/api/
+      rate_limit_msec 100    # default: 0 = no rate limiting
+      raise_on_error  false  # default: true
+      authentication  basic  # default: none
+      username        alice  # default: ''
+      password        bobpop # default: '', secret: true
+    </match>
+
+
+----
+
+Heavily based on [fluent-plugin-out-http][3]
+
+  [1]: http://fluentd.org/
+  [2]: http://open-falcon.org/
+  [3]: https://github.com/https://github.com/ento/fluent-plugin-out-http

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/lib/fluent/plugin/out_falcon.rb

@@ -0,0 +1,147 @@
+class Fluent::FalconOutput < Fluent::Output
+  Fluent::Plugin.register_output('falcon', self)
+
+  def initialize
+    super
+    require 'net/http'
+    require 'uri'
+    require 'yajl'
+  end
+
+  # Map which record field to falcon's value filed
+  #config_param :value_mapping, :string
+
+  # Endpoint URL ex. localhost.local/api/
+  config_param :endpoint_url, :string
+
+  # HTTP method
+  config_param :http_method, :string, :default => :post
+  
+  # form | json
+  config_param :serializer, :string, :default => :json
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # Raise errors that were rescued during HTTP requests?
+  config_param :raise_on_error, :bool, :default => true
+
+  # nil | 'none' | 'basic'
+  config_param :authentication, :string, :default => nil 
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => '', :secret => true
+
+  def configure(conf)
+    super
+
+    serializers = [:json, :form]
+    @serializer = if serializers.include? @serializer.intern
+                    @serializer.intern
+                  else
+                    :form
+                  end
+
+    http_methods = [:get, :put, :post, :delete]
+    @http_method = if http_methods.include? @http_method.intern
+                    @http_method.intern
+                  else
+                    :post
+                  end
+
+    @auth = case @authentication
+            when 'basic' then :basic
+            else
+              :none
+            end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    @endpoint_url
+  end
+
+  def set_body(req, tag, time, record)
+    if @serializer == :json
+      set_json_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    req
+  end
+
+  def set_json_body(req, data)
+    # TODO remove record_modifier
+    #req.body = Yajl.dump([data].map do |record|
+    #  temp = record[@value_mapping]
+    #  data.clear
+    #  data['value'] = temp
+    #  data['time'] = Time.now.to_i
+    #end)
+    req.body = Yajl.dump([data])
+    req['Content-Type'] = 'application/json'
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    return req, uri
+  end
+
+  def send_request(req, uri)    
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      $log.info('Dropped request due to rate limiting')
+      return
+    end
+    
+    res = nil
+
+    begin
+      if @auth and @auth == :basic
+        req.basic_auth(@username, @password)
+      end
+      @last_request_time = Time.now.to_f
+      res = Net::HTTP.new(uri.host, uri.port).start {|http| http.request(req) }
+    rescue => e # rescue all StandardErrors
+      # server didn't respond
+      $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
+      raise e if @raise_on_error
+    else
+       unless res and res.is_a?(Net::HTTPSuccess)
+          res_summary = if res
+                           "#{res.code} #{res.message} #{res.body}"
+                        else
+                           "res=nil"
+                        end
+          $log.warn "failed to #{req.method} #{uri} (#{res_summary})"
+       end #end unless
+    end # end begin
+  end # end send_request
+
+  def handle_record(tag, time, record)
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri)
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+    chain.next
+  end
+end

data/lib/fluent/test/falcon_output_test.rb

@@ -0,0 +1,54 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+class Test::Unit::TestCase
+end
+
+require 'webrick'
+
+# to handle POST/PUT/DELETE ...
+module WEBrick::HTTPServlet
+  class ProcHandler < AbstractServlet
+    alias do_POST   do_GET
+    alias do_PUT    do_GET
+    alias do_DELETE do_GET
+  end
+end
+
+def get_code(server, port, path, headers={})
+  require 'net/http' 
+  Net::HTTP.start(server, port){|http|
+    http.get(path, headers).code
+  } 
+end 
+def get_content(server, port, path, headers={})
+  require 'net/http'
+  Net::HTTP.start(server, port){|http|
+    http.get(path, headers).body
+  } 
+end

data/test/plugin/test_out_falcon.rb

@@ -0,0 +1,320 @@
+# -*- coding: utf-8 -*-
+require 'uri'
+require 'yajl'
+require 'fluent/test/falcon_output_test'
+require 'fluent/plugin/out_falcon'
+
+
+TEST_LISTEN_PORT = 5126
+
+
+class FalconOutputTestBase < Test::Unit::TestCase
+  # setup / teardown for servers
+  def setup
+    Fluent::Test.setup
+    @posts = []
+    @puts = []
+    @prohibited = 0
+    @requests = 0
+    @auth = false
+    @dummy_server_thread = Thread.new do
+      srv = if ENV['VERBOSE']
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
+            else
+              logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
+            end
+      begin
+        allowed_methods = %w(POST PUT)
+        srv.mount_proc('/api/') { |req,res|
+          @requests += 1
+          unless allowed_methods.include? req.request_method
+            res.status = 405
+            res.body = 'request method mismatch'
+            next
+          end
+          if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!'
+            # ok, authorized
+          elsif @auth
+            res.status = 403
+            @prohibited += 1
+            next
+          else
+            # ok, authorization not required
+          end
+
+          record = {:auth => nil}
+          if req.content_type == 'application/json'
+            record[:json] = Yajl.load(req.body)
+          else
+            record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)]
+          end
+
+          instance_variable_get("@#{req.request_method.downcase}s").push(record)
+
+          res.status = 200
+        }
+        srv.mount_proc('/') { |req,res|
+          res.status = 200
+          res.body = 'running'
+        }
+        srv.start
+      ensure
+        srv.shutdown
+      end
+    end
+
+    # to wait completion of dummy server.start()
+    require 'thread'
+    cv = ConditionVariable.new
+    watcher = Thread.new {
+      connected = false
+      while not connected
+        begin
+          get_content('localhost', TEST_LISTEN_PORT, '/')
+          connected = true
+        rescue Errno::ECONNREFUSED
+          sleep 0.1
+        rescue StandardError => e
+          p e
+          sleep 0.1
+        end
+      end
+      cv.signal
+    } 
+    mutex = Mutex.new
+    mutex.synchronize {
+      cv.wait(mutex)
+    } 
+  end
+
+  def test_dummy_server
+    host = '127.0.0.1'
+    port = TEST_LISTEN_PORT
+    client = Net::HTTP.start(host, port)
+
+    assert_equal '200', client.request_get('/').code
+    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '1', @posts[0][:form]['number']
+    assert_equal 'gauge', @posts[0][:form]['mode']
+    assert_nil @posts[0][:auth]
+
+    @auth = true
+
+    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
+
+    req_with_auth = lambda do |number, mode, user, pass|
+      url = URI.parse("http://#{host}:#{port}/api/service/metrics/pos")
+      req = Net::HTTP::Post.new(url.path)
+      req.basic_auth user, pass
+      req.set_form_data({'number'=>number, 'mode'=>mode})
+      req
+    end
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code
+
+    assert_equal 2, @posts.size
+
+  end
+
+  def teardown
+    @dummy_server_thread.kill
+    @dummy_server_thread.join
+  end
+end
+
+class FalconOutputTest < FalconOutputTestBase
+  CONFIG_FORM = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer form
+  ]
+
+  CONFIG_JSON = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer json
+  ]
+
+  CONFIG_PUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer form
+    http_method put
+  ]
+
+  CONFIG_HTTP_ERROR = %[
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+    serializer form
+  ]
+
+  CONFIG_HTTP_ERROR_SUPPRESSED = %[
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+    serializer form
+    raise_on_error false
+  ]
+
+  RATE_LIMIT_MSEC = 1200
+
+  CONFIG_RATE_LIMIT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer form
+    rate_limit_msec #{RATE_LIMIT_MSEC}
+  ]
+
+  def create_driver(conf=CONFIG_FORM, tag='test.metrics')
+    Fluent::Test::OutputTestDriver.new(Fluent::FalconOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :form, d.instance.serializer
+
+    d = create_driver CONFIG_JSON
+    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :json, d.instance.serializer
+  end
+
+  def test_emit_form
+    d = create_driver
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal '50', record[:form]['field1']
+    assert_equal '20', record[:form]['field2']
+    assert_equal '10', record[:form]['field3']
+    assert_equal '1', record[:form]['otherfield']
+    assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase
+    assert_nil record[:auth]
+
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 2, @posts.size
+  end
+
+  def test_emit_form_put
+    d = create_driver CONFIG_PUT
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @puts.size
+    record = @puts[0]
+
+    assert_equal '50', record[:form]['field1']
+    assert_nil record[:auth]
+
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @puts.size
+  end
+
+  def test_emit_json
+    binary_string = "\xe3\x81\x82".force_encoding("ascii-8bit")
+    d = create_driver CONFIG_JSON
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal 50, record[:json][0]['field1']
+    assert_equal 20, record[:json][0]['field2']
+    assert_equal 10, record[:json][0]['field3']
+    assert_equal 1, record[:json][0]['otherfield']
+    assert_equal binary_string, record[:json][0]['binary'].force_encoding("ascii-8bit")
+    assert_nil record[:auth]
+  end
+
+  def test_http_error_is_raised
+    d = create_driver CONFIG_HTTP_ERROR
+    assert_raise Errno::ECONNREFUSED do
+      d.emit({ 'field1' => 50 })
+    end
+  end
+
+  def test_http_error_is_suppressed_with_raise_on_error_false
+    d = create_driver CONFIG_HTTP_ERROR_SUPPRESSED
+    d.emit({ 'field1' => 50 })
+    d.run
+    # drive asserts the next output chain is called;
+    # so no exception means our plugin handled the error
+
+    assert_equal 0, @requests
+  end
+
+  def test_rate_limiting
+    d = create_driver CONFIG_RATE_LIMIT
+    record = { :k => 1 }
+
+    last_emit = _current_msec
+    d.emit(record)
+    d.run
+
+    assert_equal 1, @posts.size
+
+    d.emit({})
+    d.run
+    assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval"
+    assert_equal 1, @posts.size
+
+    wait_msec = 500
+    sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001
+    
+    assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
+    d.emit(record)
+    d.run
+    assert_equal 2, @posts.size
+  end
+
+  def _current_msec
+    Time.now.to_f * 1000
+  end
+  
+  def test_auth
+    @auth = true # enable authentication of dummy server
+
+    d = create_driver(CONFIG_FORM, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @prohibited
+
+    d = create_driver(CONFIG_FORM + %[
+      authentication basic
+      username alice
+      password wrong_password
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @prohibited
+
+    d = create_driver(CONFIG_FORM + %[
+      authentication basic
+      username alice
+      password secret!
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 1, @posts.size
+    assert_equal 2, @prohibited
+  end
+
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-out-falcon
+version: !ruby/object:Gem::Version
+  version: 0.1.5
+platform: ruby
+authors:
+- Marica Odagaki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: A Fluentd output plugin to send logs to falcon's push api
+email:
+- ento.entotto@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/fluent/plugin/out_falcon.rb
+- lib/fluent/test/falcon_output_test.rb
+- test/plugin/test_out_falcon.rb
+homepage: https://github.com/tsingakbar/fluent-plugin-out-falcon
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A Fluentd output plugin to send logs to falcon's push api
+test_files:
+- test/plugin/test_out_falcon.rb