fluent-plugin-out-kafka-rest 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4df0762c0b4f96b734f8492c8a4ae198260bc421
+  data.tar.gz: cc1534fdd3a60a3221d0f4259ef200917dc4cec9
+SHA512:
+  metadata.gz: db0a371008c9c3b9dbf9762570345dbcaad5d0b0446995431746b30ffe8498bef47876296d7c76bdabd1f8b66a350608cf6cdd0e7db665f4d71500355eddd417
+  data.tar.gz: 6595f68524f2c242bce7b7cb4e034d83385a87e99919fc099c05e7ec8cc52f24ce033a9c030b34f6ed26cb7b87d0bfb8486bf80dd0bdda6a68c4020c12576be0

data/.gitignore

@@ -0,0 +1,8 @@
+*~
+\#*
+.\#*
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.idea

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Changelog
+
+## 0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out-kafka-rest.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,43 @@
+# fluent-plugin-out-kafka-rest, a plugin for [Fluentd](http://fluentd.org) (WIP)
+
+A [fluentd][1] output plugin for sending logs to Kafka REST Proxy.
+This plugin does not use the native library of Apache Kafka itself.
+Please refer to Confluent's [kakfa-rest](https://github.com/confluentinc/kafka-rest)
+for the detail of REST Proxy service.
+
+## Configs
+
+    <match *>
+      type            kafka-rest
+      endpoint_url    https://localhost.local/api/
+      # use_ssl         false
+      # include_tag     false
+      # include_timestamp false
+      # http_method     post
+      # serializer      one
+      # rate_limit_msec 0
+      # authentication  basic  # default: nil
+      # username        alice
+      # password        bobpop
+    </match>
+
+## ToDo
+
+* Change tests
+* Fix the function to include tags and timestamps.
+  We should include such information into the request body.
+* Add function to submit multiple records at once.
+* Try SSL via ELB
+* Avro support
+
+## Note
+
+* Set `use_ssl` to true to use https connection
+* Set `include_tag` to true to include fluentd tag in the event log as a property 
+* Set `include_timestamp` to true to include timestamp (UNIX time) in the event log as a property
+* Set `serializer` to any to use your own serializer rule except for Kafka REST Proxy's JSON protocol
+* By default, it does not verify the https server. Use VERIFY_PEER and place the cert.pem to the location specified by OpenSSL::X509::DEFAULT_CERT_FILE. 
+* Majority of the code are cloned from  [fluent-plugin-out-https][2]
+
+  [1]: http://fluentd.org/
+  [2]: https://github.com/kazunori279/fluent-plugin-out-https

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-out-kafka-rest.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-out-kafka-rest"
+  gem.version       = "0.1.0"
+  gem.authors       = ["dobachi"]
+  gem.email         = ["dobachi1983oss@gmail.com"]
+  gem.summary       = %q{A fluentd output plugin for sending logs to Kafka REST Proxy}
+  gem.description   = gem.summary
+  gem.homepage      = "https://github.com/dobachi/fluent-plugin-out-kafka-rest"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
+  gem.add_runtime_dependency "fluentd", "~> 0.10.0"
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_kafka_rest.rb

@@ -0,0 +1,170 @@
+class Fluent::KafkaRestOutput < Fluent::Output
+  Fluent::Plugin.register_output('kafka-rest', self)
+
+  def initialize
+    super
+    require 'net/https'
+    require 'openssl'
+    require 'uri'
+    require 'yajl'
+    require 'base64'
+  end
+
+  # https or http
+  config_param :use_ssl, :bool, :default => false
+
+  # include tag
+  config_param :include_tag, :bool, :default => false
+
+  # include timestamp
+  config_param :include_timestamp, :bool, :default => false
+
+  # Endpoint URL ex. localhost.local/api/
+  config_param :endpoint_url, :string
+
+  # HTTP method
+  config_param :http_method, :string, :default => :post
+  
+  # json_bin ( Should support avro in the future)
+  config_param :serializer, :string, :default => :json_bin
+
+  # Content-Type
+  config_param :content_type, :string, :default => 'application/json'
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # nil | 'none' | 'basic'
+  config_param :authentication, :string, :default => nil 
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => ''
+
+  def configure(conf)
+    super
+
+    @use_ssl = conf['use_ssl']
+    @include_tag = conf['include_tag']
+    @include_timestamp = conf['include_timestamp']
+
+    serializers = [:json_bin]  # Should support :avro in the future
+    @serializer = if serializers.include? @serializer.intern
+                    @serializer.intern
+                  else
+                    :json_bin
+                  end
+
+    @content_type = conf['content_type']
+
+    # Kafka REST Proxy accepts only POST method at the moment
+    http_methods = [:post]
+    @http_method = if http_methods.include? @http_method.intern
+                    @http_method.intern
+                  else
+                    :post
+                  end
+
+    @auth = case @authentication
+            when 'basic' then :basic
+            else
+              :none
+            end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    @endpoint_url
+  end
+
+  def set_body(req, tag, time, record)
+    # TODO: Add avro support
+    if @include_tag
+      record['tag'] = tag
+    end
+    if @include_timestamp
+      record['timestamp'] = Time.now.to_i
+    end 
+    if @serializer == :json_bin
+      set_json_body(req, record)
+    # elsif @serializer == :avro
+    #   set_avro_body(req, record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    req
+  end
+
+  def set_json_body(req, data)
+    dumped_data = Yajl.dump(data)
+    encoded_data = Base64.encode64(dumped_data)
+    req.body = Yajl.dump({ "records" => [{ "value" => encoded_data }] })
+    req['Content-Type'] = 'application/vnd.kafka.binary.v1+json'
+  end
+
+  def set_avro_body(req, data)
+    # TODO: Implement avro body parser
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    return req, uri
+  end
+
+  def send_request(req, uri)    
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      $log.info('Dropped request due to rate limiting')
+      return
+    end
+    
+    res = nil
+    begin
+      if @auth and @auth == :basic
+        req.basic_auth(@username, @password)
+      end
+      @last_request_time = Time.now.to_f
+      https = Net::HTTP.new(uri.host, uri.port)
+      https.use_ssl = @use_ssl
+      https.ca_file = OpenSSL::X509::DEFAULT_CERT_FILE 
+#      https.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      res = https.start {|http| http.request(req) }
+    rescue IOError, EOFError, SystemCallError
+      # server didn't respond
+      $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{$!.class}, '#{$!.message}'"
+    end
+    unless res and res.is_a?(Net::HTTPSuccess)
+      res_summary = if res
+                      "#{res.code} #{res.message} #{res.body}"
+                    else
+                      "res=nil"
+                    end
+      $log.warn "failed to #{req.method} #{uri} (#{res_summary})"
+    end
+  end
+
+  def handle_record(tag, time, record)
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri)
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+    chain.next
+  end
+end

data/lib/fluent/test/https_output_test.rb

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+class Test::Unit::TestCase
+end
+
+require 'webrick'
+
+# to handle POST/PUT/DELETE ...
+module WEBrick::HTTPServlet
+  class ProcHandler < AbstractServlet
+    alias do_POST   do_GET
+    alias do_PUT    do_GET
+    alias do_DELETE do_GET
+  end
+end
+
+def get_code(server, port, path, headers={})
+  require 'net/https' 
+  https = Net::HTTP.new(server, port)
+  https.use_ssl = true
+  https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  https.start {|http|
+    http.get(path, headers).code
+  } 
+end 
+def get_content(server, port, path, headers={})
+  require 'net/https'
+  https = Net::HTTP.new(server, port)
+  https.use_ssl = true
+  https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  https.start {|http|
+    http.get(path, headers).body
+  } 
+end

data/test/plugin/test_out_https.rb

@@ -0,0 +1,270 @@
+# -*- coding: utf-8 -*-
+require 'net/https'
+require 'uri'
+require 'yajl'
+require 'fluent/test/https_output_test'
+require 'fluent/plugin/out_kafka_rest'
+require 'webrick'
+require 'webrick/https'
+require 'base64'
+
+TEST_LISTEN_PORT = 5126
+
+
+class HTTPSOutputTestBase < Test::Unit::TestCase
+  # setup / teardown for servers
+  def setup
+    Fluent::Test.setup
+    @posts = []
+    @puts = []
+    @prohibited = 0
+    @requests = 0
+    @auth = false
+    @dummy_server_thread = Thread.new do
+      srv = if ENV['VERBOSE']
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, SSLEnable => true, :SSLCertName  => [ [ 'CN', WEBrick::Utils::getservername ] ]})
+            else
+              logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => [], :SSLEnable => true, :SSLCertName  => [ [ 'CN', WEBrick::Utils::getservername ] ]})
+            end
+      begin
+        # Kafka REST Proxy accepts only POST method at the moment
+        allowed_methods = %w(POST)
+        srv.mount_proc('/api/') { |req,res|
+          @requests += 1
+          unless allowed_methods.include? req.request_method
+            res.status = 405
+            res.body = 'request method mismatch'
+            next
+          end
+          if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!'
+            # ok, authorized
+          elsif @auth
+            res.status = 403
+            @prohibited += 1
+            next
+          else
+            # ok, authorization not required
+          end
+
+          record = {:auth => nil}
+          if req.content_type == 'application/vnd.kafka.binary.v1+json'
+            raw_content = Yajl.load(req.body)
+            record[:body] = raw_content["records"].map{|s| Base64.decode64(s["value"])}
+          else
+            record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)]
+          end
+
+          instance_variable_get("@#{req.request_method.downcase}s").push(record)
+
+          res.status = 200
+        }
+        srv.mount_proc('/') { |req,res|
+          res.status = 200
+          res.body = 'running'
+        }
+        srv.start
+      ensure
+        srv.shutdown
+      end
+    end
+
+    # to wait completion of dummy server.start()
+    require 'thread'
+    cv = ConditionVariable.new
+    watcher = Thread.new {
+      connected = false
+      while not connected
+        begin
+          get_content('localhost', TEST_LISTEN_PORT, '/')
+          connected = true
+        rescue Errno::ECONNREFUSED
+          sleep 0.1
+        rescue StandardError => e
+          p e
+          sleep 0.1
+        end
+      end
+      cv.signal
+    } 
+    mutex = Mutex.new
+    mutex.synchronize {
+      cv.wait(mutex)
+    } 
+  end
+
+  def test_dummy_server
+    host = '127.0.0.1'
+    port = TEST_LISTEN_PORT
+    https = Net::HTTP.new(host, port)
+    https.use_ssl = true
+    https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    client = https.start() 
+
+    assert_equal '200', client.request_get('/').code
+    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '1', @posts[0][:form]['number']
+    assert_equal 'gauge', @posts[0][:form]['mode']
+    assert_nil @posts[0][:auth]
+
+    @auth = true
+
+    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
+
+    req_with_auth = lambda do |number, mode, user, pass|
+      url = URI.parse("https://#{host}:#{port}/api/service/metrics/pos")
+      req = Net::HTTP::Post.new(url.path)
+      req.basic_auth user, pass
+      req.set_form_data({'number'=>number, 'mode'=>mode})
+      req
+    end
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code
+
+    assert_equal 2, @posts.size
+
+  end
+
+  def teardown
+    @dummy_server_thread.kill
+    @dummy_server_thread.join
+  end
+end
+
+class HTTPSOutputTest < HTTPSOutputTestBase
+  CONFIG = %[
+    use_ssl true
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+  ]
+
+  CONFIG_JSON_BIN = %[
+    use_ssl true
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer json_bin
+  ]
+
+  CONFIG_HTTP_ERROR = %[
+    use_ssl true
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+  ]
+
+  RATE_LIMIT_MSEC = 1200
+
+  CONFIG_RATE_LIMIT = %[
+    use_ssl true
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    rate_limit_msec #{RATE_LIMIT_MSEC}
+  ]
+
+  def create_driver(conf=CONFIG, tag='test.metrics')
+    Fluent::Test::OutputTestDriver.new(Fluent::KafkaRestOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal "https://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :json_bin, d.instance.serializer
+
+    d = create_driver CONFIG_JSON_BIN
+    assert_equal "https://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :json_bin, d.instance.serializer
+  end
+
+  def test_emit_json_bin
+    binary_string = "\xe3\x81\x82".force_encoding("ascii-8bit")
+    d = create_driver CONFIG_JSON_BIN
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+    body = Yajl.load(record[:body][0])
+
+    assert_equal 50, body['field1']
+    assert_equal 20, body['field2']
+    assert_equal 10, body['field3']
+    assert_equal 1, body['otherfield']
+    assert_equal binary_string, body['binary']
+    assert_nil record[:auth]
+  end
+
+  def test_http_error
+    d = create_driver CONFIG_HTTP_ERROR
+    d.emit({ 'field1' => 50 })
+    d.run
+    # drive asserts the next output chain is called;
+    # so no exception means our plugin handled the error
+
+    assert_equal 0, @requests
+  end
+
+  def test_rate_limiting
+    d = create_driver CONFIG_RATE_LIMIT
+    record = { :k => 1 }
+
+    last_emit = _current_msec
+    d.emit(record)
+    d.run
+
+    assert_equal 1, @posts.size
+
+    d.emit({})
+    d.run
+    assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval"
+    assert_equal 1, @posts.size
+
+    sleep (last_emit + RATE_LIMIT_MSEC - _current_msec) * 0.001
+    
+    assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
+    d.emit(record)
+    d.run
+    assert_equal 2, @posts.size
+  end
+
+  def _current_msec
+    Time.now.to_f * 1000
+  end
+  
+  def test_auth
+    @auth = true # enable authentication of dummy server
+
+    d = create_driver(CONFIG, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @prohibited
+
+    d = create_driver(CONFIG + %[
+      authentication basic
+      username alice
+      password wrong_password
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @prohibited
+
+    d = create_driver(CONFIG + %[
+      authentication basic
+      username alice
+      password secret!
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 1, @posts.size
+    assert_equal 2, @prohibited
+  end
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-out-kafka-rest
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- dobachi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fluentd output plugin for sending logs to Kafka REST Proxy
+email:
+- dobachi1983oss@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-out-kafka-rest.gemspec
+- lib/fluent/plugin/out_kafka_rest.rb
+- lib/fluent/test/https_output_test.rb
+- test/plugin/test_out_https.rb
+homepage: https://github.com/dobachi/fluent-plugin-out-kafka-rest
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.5
+signing_key: 
+specification_version: 4
+summary: A fluentd output plugin for sending logs to Kafka REST Proxy
+test_files:
+- test/plugin/test_out_https.rb