fluent-plugin-out-falcon 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93262b579cc60c929f865d81171a46b4d610738d
-  data.tar.gz: 6d0e4885faaebce8db8a1ca12488f9f0fd426a3c
+  metadata.gz: 7e69f38952bf8c9cf7da9555ffaedd7c83f7b47d
+  data.tar.gz: 5e1313a65375d800cfc35d73442748dee87911c0
 SHA512:
-  metadata.gz: 00d5bc6bb7e9ed15148858bc792a3aed8290cea197e7f66c8d22cfe2579667583df53e79edcbd8a3b2c7ff3b7e2e026b184ddc48f063703c8d8e51c2b1764aef
-  data.tar.gz: 6d8d6d3b27bc2cdbca11cd8238d127a90ecc5b14337bad1360d479e0e5bb5abb8d7bd62549b3b3a0610ce1fed8a8ec217193de2cd39bbfdc2bb811c52f664d27
+  metadata.gz: 78e8a8af151ac7e71d86c9f83de59aca0e986dadf16758e38e857fd84d6131524afa5e078bcde962c77ebf114fa47f69dfe7f31efde435ca85b0d27a62c68a87
+  data.tar.gz: f0f3564531cd9e50af2b50a21a2c38d32e0ff930ef4c77d24bfbc7ce0c1dfdc20137e7338f05a31d372fa0ad4a3b9de7b7c2a978ab686f35e34a1b6497221d6b

data/README.md

@@ -1,12 +1,23 @@
 # fluent-plugin-out-falcon, a plugin for [Fluentd](http://fluentd.org)
 
-A [fluentd][1] output plugin for sending logs to [falcon][2]'s push api.
+A [fluentd][1] output plugin for sending logs to [falcon][2]'s push API.
 
 ## Configuration options
 
-    <match *>
+`record` option will be posted to falcon.
+It can only be in one line(limited by fluentd), in the form as `([{ruby_hash_items},{ruby_hash_items},...])`.
+The outsider `()` is used to avoid fluentd interpret this line as json.
+You can use `tag`, `time` and `record` variables while writing ruby\_hash.
+
+    <source>
+      @type dummy
+      tag xx.yyy
+      dummy {"ip":"8.8.8.8", "data1":888, "data2": "message"} 
+    </source>
+    <match xx.yyy>
       type falcon
-      endpoint_url    http://localhost.local/api/
+      endpoint_url    http://localhost:1988/v1/push
+      records         ([{'metric'=>'number_data', 'endpoint'=>record['ip'], 'timestamp'=>time, 'value'=>record['data1'], 'step'=>1, 'counterType'=>'GAUGE', 'tags'=>nil}, {'metric'=>'string_data', 'endpoint'=>record['ip'], 'timestamp'=>time, 'value'=>record['data2'], 'step'=>1, 'counterType'=>'GAUGE', 'tags'=>nil}])
       rate_limit_msec 100    # default: 0 = no rate limiting
       raise_on_error  false  # default: true
       authentication  basic  # default: none

data/fluent-plugin-out-falcon.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-out-falcon"
+  gem.version       = "0.1.6"
+  gem.authors       = ["tsingakbar"]
+  gem.email         = ["yuqing@cmcm.com"]
+  gem.summary       = %q{A Fluentd output plugin to send logs to falcon's push API}
+  gem.description   = gem.summary
+  gem.homepage      = "https://github.com/tsingakbar/fluent-plugin-out-falcon"
+  gem.licenses      = ["Apache-2.0"]
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.10.0", "< 2"]
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", ">= 3.1.0"
+end

data/lib/fluent/plugin/out_falcon.rb

@@ -8,18 +8,11 @@ class Fluent::FalconOutput < Fluent::Output
     require 'yajl'
   end
 
-  # Map which record field to falcon's value filed
-  #config_param :value_mapping, :string
+  config_param :records, :string
 
   # Endpoint URL ex. localhost.local/api/
   config_param :endpoint_url, :string
 
-  # HTTP method
-  config_param :http_method, :string, :default => :post
-  
-  # form | json
-  config_param :serializer, :string, :default => :json
-
   # Simple rate limiting: ignore any records within `rate_limit_msec`
   # since the last one.
   config_param :rate_limit_msec, :integer, :default => 0
@@ -35,20 +28,6 @@ class Fluent::FalconOutput < Fluent::Output
   def configure(conf)
     super
 
-    serializers = [:json, :form]
-    @serializer = if serializers.include? @serializer.intern
-                    @serializer.intern
-                  else
-                    :form
-                  end
-
-    http_methods = [:get, :put, :post, :delete]
-    @http_method = if http_methods.include? @http_method.intern
-                    @http_method.intern
-                  else
-                    :post
-                  end
-
     @auth = case @authentication
             when 'basic' then :basic
             else
@@ -69,11 +48,8 @@ class Fluent::FalconOutput < Fluent::Output
   end
 
   def set_body(req, tag, time, record)
-    if @serializer == :json
-      set_json_body(req, record)
-    else
-      req.set_form_data(record)
-    end
+    req.body = Yajl.dump(eval(@records))
+    req['Content-Type'] = 'application/json'
     req
   end
 
@@ -81,22 +57,10 @@ class Fluent::FalconOutput < Fluent::Output
     req
   end
 
-  def set_json_body(req, data)
-    # TODO remove record_modifier
-    #req.body = Yajl.dump([data].map do |record|
-    #  temp = record[@value_mapping]
-    #  data.clear
-    #  data['value'] = temp
-    #  data['time'] = Time.now.to_i
-    #end)
-    req.body = Yajl.dump([data])
-    req['Content-Type'] = 'application/json'
-  end
-
   def create_request(tag, time, record)
     url = format_url(tag, time, record)
     uri = URI.parse(url)
-    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+    req = Net::HTTP.const_get('Post').new(uri.path)
     set_body(req, tag, time, record)
     set_header(req, tag, time, record)
     return req, uri

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-falcon
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
-- Marica Odagaki
+- tsingakbar
 autorequire: 
 bindir: bin
 cert_chain: []
@@ -86,9 +86,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
-description: A Fluentd output plugin to send logs to falcon's push api
+description: A Fluentd output plugin to send logs to falcon's push API
 email:
-- ento.entotto@gmail.com
+- yuqing@cmcm.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -98,10 +98,8 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
+- fluent-plugin-out-falcon.gemspec
 - lib/fluent/plugin/out_falcon.rb
-- lib/fluent/test/falcon_output_test.rb
-- test/plugin/test_out_falcon.rb
 homepage: https://github.com/tsingakbar/fluent-plugin-out-falcon
 licenses:
 - Apache-2.0
@@ -125,6 +123,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: A Fluentd output plugin to send logs to falcon's push api
-test_files:
-- test/plugin/test_out_falcon.rb
+summary: A Fluentd output plugin to send logs to falcon's push API
+test_files: []

data/Rakefile

@@ -1,11 +0,0 @@
-#!/usr/bin/env rake
-require "bundler/gem_tasks"
-
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-task :default => :test

data/lib/fluent/test/falcon_output_test.rb

@@ -1,54 +0,0 @@
-require 'rubygems'
-require 'bundler'
-
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-
-require 'test/unit'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-
-require 'fluent/test'
-
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
-class Test::Unit::TestCase
-end
-
-require 'webrick'
-
-# to handle POST/PUT/DELETE ...
-module WEBrick::HTTPServlet
-  class ProcHandler < AbstractServlet
-    alias do_POST   do_GET
-    alias do_PUT    do_GET
-    alias do_DELETE do_GET
-  end
-end
-
-def get_code(server, port, path, headers={})
-  require 'net/http' 
-  Net::HTTP.start(server, port){|http|
-    http.get(path, headers).code
-  } 
-end 
-def get_content(server, port, path, headers={})
-  require 'net/http'
-  Net::HTTP.start(server, port){|http|
-    http.get(path, headers).body
-  } 
-end

data/test/plugin/test_out_falcon.rb

@@ -1,320 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'uri'
-require 'yajl'
-require 'fluent/test/falcon_output_test'
-require 'fluent/plugin/out_falcon'
-
-
-TEST_LISTEN_PORT = 5126
-
-
-class FalconOutputTestBase < Test::Unit::TestCase
-  # setup / teardown for servers
-  def setup
-    Fluent::Test.setup
-    @posts = []
-    @puts = []
-    @prohibited = 0
-    @requests = 0
-    @auth = false
-    @dummy_server_thread = Thread.new do
-      srv = if ENV['VERBOSE']
-              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
-            else
-              logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
-              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
-            end
-      begin
-        allowed_methods = %w(POST PUT)
-        srv.mount_proc('/api/') { |req,res|
-          @requests += 1
-          unless allowed_methods.include? req.request_method
-            res.status = 405
-            res.body = 'request method mismatch'
-            next
-          end
-          if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!'
-            # ok, authorized
-          elsif @auth
-            res.status = 403
-            @prohibited += 1
-            next
-          else
-            # ok, authorization not required
-          end
-
-          record = {:auth => nil}
-          if req.content_type == 'application/json'
-            record[:json] = Yajl.load(req.body)
-          else
-            record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)]
-          end
-
-          instance_variable_get("@#{req.request_method.downcase}s").push(record)
-
-          res.status = 200
-        }
-        srv.mount_proc('/') { |req,res|
-          res.status = 200
-          res.body = 'running'
-        }
-        srv.start
-      ensure
-        srv.shutdown
-      end
-    end
-
-    # to wait completion of dummy server.start()
-    require 'thread'
-    cv = ConditionVariable.new
-    watcher = Thread.new {
-      connected = false
-      while not connected
-        begin
-          get_content('localhost', TEST_LISTEN_PORT, '/')
-          connected = true
-        rescue Errno::ECONNREFUSED
-          sleep 0.1
-        rescue StandardError => e
-          p e
-          sleep 0.1
-        end
-      end
-      cv.signal
-    } 
-    mutex = Mutex.new
-    mutex.synchronize {
-      cv.wait(mutex)
-    } 
-  end
-
-  def test_dummy_server
-    host = '127.0.0.1'
-    port = TEST_LISTEN_PORT
-    client = Net::HTTP.start(host, port)
-
-    assert_equal '200', client.request_get('/').code
-    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
-
-    assert_equal 1, @posts.size
-
-    assert_equal '1', @posts[0][:form]['number']
-    assert_equal 'gauge', @posts[0][:form]['mode']
-    assert_nil @posts[0][:auth]
-
-    @auth = true
-
-    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
-
-    req_with_auth = lambda do |number, mode, user, pass|
-      url = URI.parse("http://#{host}:#{port}/api/service/metrics/pos")
-      req = Net::HTTP::Post.new(url.path)
-      req.basic_auth user, pass
-      req.set_form_data({'number'=>number, 'mode'=>mode})
-      req
-    end
-
-    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
-
-    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
-
-    assert_equal 1, @posts.size
-
-    assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code
-
-    assert_equal 2, @posts.size
-
-  end
-
-  def teardown
-    @dummy_server_thread.kill
-    @dummy_server_thread.join
-  end
-end
-
-class FalconOutputTest < FalconOutputTestBase
-  CONFIG_FORM = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
-    serializer form
-  ]
-
-  CONFIG_JSON = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
-    serializer json
-  ]
-
-  CONFIG_PUT = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
-    serializer form
-    http_method put
-  ]
-
-  CONFIG_HTTP_ERROR = %[
-    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
-    serializer form
-  ]
-
-  CONFIG_HTTP_ERROR_SUPPRESSED = %[
-    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
-    serializer form
-    raise_on_error false
-  ]
-
-  RATE_LIMIT_MSEC = 1200
-
-  CONFIG_RATE_LIMIT = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
-    serializer form
-    rate_limit_msec #{RATE_LIMIT_MSEC}
-  ]
-
-  def create_driver(conf=CONFIG_FORM, tag='test.metrics')
-    Fluent::Test::OutputTestDriver.new(Fluent::FalconOutput, tag).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
-    assert_equal :form, d.instance.serializer
-
-    d = create_driver CONFIG_JSON
-    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
-    assert_equal :json, d.instance.serializer
-  end
-
-  def test_emit_form
-    d = create_driver
-    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") })
-    d.run
-
-    assert_equal 1, @posts.size
-    record = @posts[0]
-
-    assert_equal '50', record[:form]['field1']
-    assert_equal '20', record[:form]['field2']
-    assert_equal '10', record[:form]['field3']
-    assert_equal '1', record[:form]['otherfield']
-    assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase
-    assert_nil record[:auth]
-
-    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
-    d.run
-
-    assert_equal 2, @posts.size
-  end
-
-  def test_emit_form_put
-    d = create_driver CONFIG_PUT
-    d.emit({ 'field1' => 50 })
-    d.run
-
-    assert_equal 0, @posts.size
-    assert_equal 1, @puts.size
-    record = @puts[0]
-
-    assert_equal '50', record[:form]['field1']
-    assert_nil record[:auth]
-
-    d.emit({ 'field1' => 50 })
-    d.run
-
-    assert_equal 0, @posts.size
-    assert_equal 2, @puts.size
-  end
-
-  def test_emit_json
-    binary_string = "\xe3\x81\x82".force_encoding("ascii-8bit")
-    d = create_driver CONFIG_JSON
-    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
-    d.run
-
-    assert_equal 1, @posts.size
-    record = @posts[0]
-
-    assert_equal 50, record[:json][0]['field1']
-    assert_equal 20, record[:json][0]['field2']
-    assert_equal 10, record[:json][0]['field3']
-    assert_equal 1, record[:json][0]['otherfield']
-    assert_equal binary_string, record[:json][0]['binary'].force_encoding("ascii-8bit")
-    assert_nil record[:auth]
-  end
-
-  def test_http_error_is_raised
-    d = create_driver CONFIG_HTTP_ERROR
-    assert_raise Errno::ECONNREFUSED do
-      d.emit({ 'field1' => 50 })
-    end
-  end
-
-  def test_http_error_is_suppressed_with_raise_on_error_false
-    d = create_driver CONFIG_HTTP_ERROR_SUPPRESSED
-    d.emit({ 'field1' => 50 })
-    d.run
-    # drive asserts the next output chain is called;
-    # so no exception means our plugin handled the error
-
-    assert_equal 0, @requests
-  end
-
-  def test_rate_limiting
-    d = create_driver CONFIG_RATE_LIMIT
-    record = { :k => 1 }
-
-    last_emit = _current_msec
-    d.emit(record)
-    d.run
-
-    assert_equal 1, @posts.size
-
-    d.emit({})
-    d.run
-    assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval"
-    assert_equal 1, @posts.size
-
-    wait_msec = 500
-    sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001
-    
-    assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
-    d.emit(record)
-    d.run
-    assert_equal 2, @posts.size
-  end
-
-  def _current_msec
-    Time.now.to_f * 1000
-  end
-  
-  def test_auth
-    @auth = true # enable authentication of dummy server
-
-    d = create_driver(CONFIG_FORM, 'test.metrics')
-    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
-    d.run # failed in background, and output warn log
-
-    assert_equal 0, @posts.size
-    assert_equal 1, @prohibited
-
-    d = create_driver(CONFIG_FORM + %[
-      authentication basic
-      username alice
-      password wrong_password
-    ], 'test.metrics')
-    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
-    d.run # failed in background, and output warn log
-
-    assert_equal 0, @posts.size
-    assert_equal 2, @prohibited
-
-    d = create_driver(CONFIG_FORM + %[
-      authentication basic
-      username alice
-      password secret!
-    ], 'test.metrics')
-    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
-    d.run # failed in background, and output warn log
-
-    assert_equal 1, @posts.size
-    assert_equal 2, @prohibited
-  end
-
-end