fluent-plugin-http-puma 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 57610401c51ad5e48c1a29aa19a1efa383d65b55
+  data.tar.gz: a9d505153b256aae456c914f367621f7e0fd1b12
+SHA512:
+  metadata.gz: 458ccbac41fe5c06edc1e64a7fcbc01e65e83f2276023e950313e24c830577e9f01d7a0e36434e7de6bee6bd45cddb210e362fa7c74e043624b48a1aef00cf5a
+  data.tar.gz: e6e367fe5cf85ca046666dedee81ada2ad8b557f5bc287725bd05234f8e3c58e590175637728bba2275ce423efb094b2d38e09d4d43b15f6702329fbd1d324d4

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - ruby-head
+  - rbx-2
+
+branches:
+  only:
+    - master
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx-2
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,40 @@
+# Puma based HTTP(S) plugin for Fluentd
+
+HTTP input plugin based on [Puma](http://puma.io/)
+
+This HTTP input plugin supports HTTPS mode.
+
+## Installation
+
+Use RubyGems:
+
+    fluent-gem install fluent-plugin-http-puma
+
+## Configuration
+
+    <source>
+      type http_puma
+
+      # optional parameters(same as in_http)
+      bind 127.0.0.1
+      port 9890
+      backlog 2048
+      format json
+
+      # optional Puma parameters
+      min_threads 0
+      max_threads 4
+      use_ssl
+      ssl_keys /path/to/key,/path/to/cert
+    </match>
+
+### HTTPS mode
+
+Use `use_ssl` and `ssl_keys`.
+
+    <source>
+      use_ssl
+      ssl_keys ["/path/to/key", "/path/to/cert"] # ssl_keys is required when use_ssl is true
+    </source>
+
+`ssl_keys` are json array which has two elements, 1st is for key file, 2nd is for cert file.

data/Rakefile

@@ -0,0 +1,15 @@
+#!/usr/bin/env rake
+
+require 'bundler'
+require 'bundler/gem_tasks'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = (Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/fluent-plugin-http-puma.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-http-puma"
+  gem.description = "Puma based HTTP(S) plugin for Fluentd"
+  gem.homepage    = "https://github.com/repeatedly/fluent-plugin-http-puma"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Masahiro Nakagawa"]
+  gem.email       = "repeatedly@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'Apache License (2.0)'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.50"
+  gem.add_dependency "puma", "~> 2.9.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/in_http_puma.rb

@@ -0,0 +1,178 @@
+module Fluent
+  class HttpPumaInput < Input
+    Plugin.register_input('http_puma', self)
+
+    def initialize
+      require 'puma'
+      require 'uri'
+      super
+    end
+
+    config_param :port, :integer, :default => 9890
+    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :min_threads, :integer, :default => 0
+    config_param :max_threads, :integer, :default => 4
+    config_param :use_ssl, :bool, :default => false
+    config_param :ssl_keys, :array, :default => nil
+    config_param :backlog, :integer, :default => nil
+    config_param :format, :string, :default => 'default'
+
+    def configure(conf)
+      super
+
+      if @use_ssl && !@ssl_keys
+        raise ConfigError, 'ssl_keys parameter is required when use_ssl is true'
+      end
+
+      if @format != 'default'
+        @parser = TextParser.new
+        @parser.configure(conf)
+      end
+    end
+
+    def start
+      super
+
+      # Refer puma's Runner and Rack handler for puma server setup
+      @server = ::Puma::Server.new(method(:on_request))
+      @server.min_threads = @min_threads
+      @server.max_threads = @max_threads
+      @server.leak_stack_on_error = false
+      if @use_ssl
+        setup_https
+      else
+        setup_http
+      end
+
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @server.stop(true)
+      @thread.join
+    end
+
+    def run
+      @server.run(false)
+    rescue => e
+      log.error "unexpected error", :error => e.to_s
+      log.error_backtrace e.backtrace
+    end
+
+    OK_RESPONSE = [200, {'Content-type'=>'text/plain'}, ["OK"]]
+
+    def on_request(env)
+      uri = URI.parse(env['REQUEST_URI'.freeze])
+      params = Rack::Utils.parse_query(uri.query)
+      path_info = uri.path
+
+      begin
+        path = path_info[1..-1]  # remove /
+        tag = path.split('/').join('.')
+        record_time, record = parse_record(env, params)
+
+        # Skip nil record
+        if record.nil?
+          return OK_RESPONSE
+        end
+
+        time = if param_time = params['time'.freeze]
+                 param_time = param_time.to_i
+                 param_time.zero? ? Engine.now : param_time
+               else
+                 record_time.nil? ? Engine.now : record_time
+               end
+      rescue => e
+        return [400, {'Content-type'=>'text/plain'}, ["Bad Request\n#{e}\n"]]
+      end
+
+      begin
+        # Support batched requests
+        if record.is_a?(Array)           
+          mes = MultiEventStream.new
+          record.each do |single_record|
+            single_time = single_record.delete("time".freeze) || time 
+            mes.add(single_time, single_record)
+          end
+          Engine.emit_stream(tag, mes)
+	else
+          Engine.emit(tag, time, record)
+        end
+      rescue => e
+        return [500, {'Content-type'=>'text/plain'}, ["Internal Server Error\n#{e}\n"]]
+      end
+
+      return OK_RESPONSE
+    end
+
+    private
+
+    def setup_http
+      log.info "listening http on #{@bind}:#{@port}"
+
+      opts = [@bind, @port, true]
+      opts << @backlog if @backlog
+      @server.add_tcp_listener(*opts)
+    end
+
+    def setup_https
+      require 'puma/minissl'
+
+      ctx = ::Puma::MiniSSL::Context.new
+      key, cert = @ssl_keys
+      if defined?(JRUBY_VERSION)
+        ctx.keystore = key
+        ctx.keystore_pass = cert
+      else
+        ctx.key = key
+        ctx.cert = cert
+      end
+      ctx.verify_mode = ::Puma::MiniSSL::VERIFY_PEER
+      #ctx.verify_mode = ::Puma::MiniSSL::VERIFY_NONE
+
+      log.info "listening https on #{@bind}:#{@port}"
+
+      opts = [@bind, @port, ctx, true]
+      opts << @backlog if @backlog
+      @server.add_ssl_listener(*opts)
+    end
+
+    def parse_record(env, params)
+      body = env['rack.input'.freeze]
+      content_type = env['CONTENT_TYPE'.freeze]
+
+      case
+      when @format != 'default'.freeze
+        parse_params_with_parser(body.read)
+      when content_type.start_with?('application/json'.freeze)
+        return nil, JSON.load(body)
+      when content_type.start_with?('application/x-msgpack'.freeze)
+        return nil, MessagePack.unpack(body)
+      when content_type =~ /^application\/x-www-form-urlencoded/
+        params.update(::Rack::Utils.parse_query(body.read))
+        parse_params_form(params)
+      when content_type =~ /^multipart\/form-data; boundary=(.+)/
+        params.update(::Rack::Multipart.parse_multipart(env))
+        parse_params_form(params)
+      end
+    end
+
+    def parse_params_form(params)
+      record = if msgpack = params['msgpack'.freeze]
+                 MessagePack.unpack(msgpack)
+               elsif js = params['json'.freeze]
+                 JSON.parse(js)
+               else
+                 raise "'json' or 'msgpack' parameter is required"
+               end
+      return nil, record
+    end
+
+    def parse_params_with_parser(content)
+      @parser.parse(content) { |time, record|
+        raise "received event is not #{@format}: #{content}" if record.nil?
+        return time, record
+      }
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'test/unit'
+require 'fileutils'
+require 'fluent/log'
+require 'fluent/test'
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
+
+$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)

data/test/plugin/cert_puma.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/jCCAeagAwIBAgIBAjANBgkqhkiG9w0BAQUFADA5MQswCQYDVQQGEwJVUzEO
+MAwGA1UECgwFbG9jYWwxDTALBgNVBAsMBGFlcm8xCzAJBgNVBAMMAkNBMB4XDTEy
+MDExNDAwMjcyN1oXDTEzMDExMzAwMjcyN1owSDELMAkGA1UEBhMCVVMxDjAMBgNV
+BAoMBWxvY2FsMQ0wCwYDVQQLDARhZXJvMQswCQYDVQQLDAJDQTENMAsGA1UEAwwE
+cHVtYTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEArxfNMp+g/pKhsDEkB3KR
+1MAkbfnN/UKMvfXwlnXpz7YX1LHHnMutiI/PqymAp6BPcu+umuW2qMHQyqqtyATm
+Z9jr3t837nhmxwG1noRaKRtsckn9FD43ZlpPg0Q5QnhS4oOsXwJzilqPjdDFYrKN
+3TSvIGM2+hVqpVoGYAHDKbMCAwEAAaOBhTCBgjAMBgNVHRMBAf8EAjAAMDEGCWCG
+SAGG+EIBDQQkFiJSdWJ5L09wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0G
+A1UdDgQWBBTyDyJlmYBDwfWdRj6lWGvoY43k9DALBgNVHQ8EBAMCBaAwEwYDVR0l
+BAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQEFBQADggEBAIbBVfoVCG8RyVesPW+q
+5i0wAMbHZ1fwv1RKp17c68DYDs0YYPi0bA0ss8AgpU6thWmskxPiFaE6D5x8iv9f
+zkcHxgr1Mrbx6RLx9tLUVehSmRv3aiVO4k9Mp6vf+rJK1AYeaGBmvoqTBLwy7Jrt
+ytKMdqMJj5jKWkWgEGgTnjzbcOClmCQab9isigIzTxMyC/LjeKZe8pPeVX6OM8bY
+y8XGZp9B7uwdPzqt/g25IzTC0KsQwq8cB0raAtZzIyTNv42zcUjmQNVazAozCTcq
+MsEtK2z7TYBC3udTsdyS2qVqCpsk7IMOBGrw8vk4SNhO+coiDObW2K/HNvhl0tZC
+oQI=
+-----END CERTIFICATE-----

data/test/plugin/puma_keypair.pem

@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQCvF80yn6D+kqGwMSQHcpHUwCRt+c39Qoy99fCWdenPthfUscec
+y62Ij8+rKYCnoE9y766a5baowdDKqq3IBOZn2Ove3zfueGbHAbWehFopG2xySf0U
+PjdmWk+DRDlCeFLig6xfAnOKWo+N0MViso3dNK8gYzb6FWqlWgZgAcMpswIDAQAB
+AoGAHv/UyZivdULas4oPue3T2dnm2T239ZXZuywW21ym96pij7ql/6Gj6KClgMVJ
+TOQ6DLxYqn3vF/OwlqEfQWF0tTUYY+xNbEDE1YsbrS5/FSzbaEYYOHzRl/vMmnsf
+aNgYaSjOIecin7L71Wzq0piMIxg8BLb6IVECBku9EQNzxuECQQDZsbRgg1XZGj+r
+XAu/qXTNKQ/r7k+iPN5bXON6ApBomG+4Q7VVITL3tkGzLOphRZ37Q28FrN4B4gtC
+Xb9il5lDAkEAzecTSopPi2VdcME4WWmwn1rbTp/jJNt4dGZLsNfj9RejVDd32i/L
+P7wCpoPDaaVcoF2HgvCs39qatyVg6ecu0QJBALN4q+q9nDMGTuNpWU5D2EWjyrqJ
+mCF66R6NcASQxJlWwxQ4zfBHFIvgOD4Nk5VqHZqet5MIN2d6AipOu4/+x50CQHDp
+jf+rd1GHBcXGf8MwnUXWCjvEnEhi/lw+mLVivsRx8QRG4rfIy9monX949Flj8DaU
+87IPj422kG9s1QeP2nECQQCkg+RUcoQm7SiM8OXuXNeHQlvQNp65geFRxzKAXxT/
++1Mbtwnd3AXXZBekFDDpE9U3ZQjahoe7oc1oUBuw5hXL
+-----END RSA PRIVATE KEY-----

data/test/plugin/test_in_http_puma.rb

@@ -0,0 +1,209 @@
+require 'helper'
+require 'fluent/plugin/in_http_puma'
+require 'net/https'
+
+class HttpPumaInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind "127.0.0.1"
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::HttpPumaInput).configure(conf, true)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal PORT, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal false, d.instance.use_ssl
+    assert_equal 0, d.instance.min_threads
+    assert_equal 4, d.instance.max_threads
+  end
+
+  def test_time
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      d.expected_emits.each { |tag, time, record|
+        res = post("/#{tag}", {"json" => record.to_json})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      d.expected_emits.each { |tag,time,record|
+        res = post("/#{tag}", {"json" => record.to_json, "time" => time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  # puma_keypair.pem and cert_puma.pem are from puma's repository
+  def test_json_over_https
+    dir = File.dirname(__FILE__)
+    d = create_driver(CONFIG + %[
+      use_ssl
+      ssl_keys ["#{File.join(dir, 'puma_keypair.pem')}", "#{File.join(dir, 'cert_puma.pem')}"]
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      d.expected_emits.each { |tag,time,record|
+        res = post("/#{tag}", {"json" => record.to_json, "time" => time.to_s}, {}, true)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_multi_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [{"a" => 1},{"a" => 2}]
+    tag = "tag1"
+
+    events.each { |ev|
+      d.expect_emit tag, time, ev
+    }
+
+    d.run do
+      res = post("/#{tag}", {"json" => events.to_json, "time" => time.to_s})
+      assert_equal "200", res.code
+    end
+  end
+
+  def test_application_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}?time=#{time.to_s}", record.to_json, {"content-type" => "application/json; charset=utf-8"})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}", {"msgpack" => record.to_msgpack, "time" => time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_multi_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [{"a" => 1},{"a" => 2}]
+    tag = "tag1"
+
+    events.each { |ev|
+      d.expect_emit tag, time, ev
+    }
+
+    d.run do
+      res = post("/#{tag}", {"msgpack" => events.to_msgpack, "time" => time.to_s})
+      assert_equal "200", res.code
+    end
+
+  end
+
+  def test_with_regexp
+    d = create_driver(CONFIG + %[
+      format /^(?<field_1>\\\\d+):(?<field_2>\\\\w+)$/
+      types field_1:integer
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"field_1" => 1, "field_2" => 'str'}
+    d.expect_emit "tag2", time, {"field_1" => 2, "field_2" => 'str'}
+
+    d.run do
+      d.expected_emits.each { |tag, time, record|
+        body = record.map { |k, v|
+          v.to_s
+        }.join(':')
+        res = post("/#{tag}?time=#{time.to_s}", body)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_with_csv
+    require 'csv'
+
+    d = create_driver(CONFIG + %[
+      format csv
+      keys foo,bar
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"foo" => "1", "bar" => 'st"r'}
+    d.expect_emit "tag2", time, {"foo" => "2", "bar" => 'str'}
+
+    d.run do
+      d.expected_emits.each { |tag, time, record|
+        body = record.map { |k, v| v }.to_csv
+        res = post("/#{tag}?time=#{time.to_s}", body)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def post(path, params, header = {}, ssl = false)
+    http = Net::HTTP.new("127.0.0.1", PORT)
+    if ssl
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    req = Net::HTTP::Post.new(path, header)
+    if params.is_a?(String)
+      req.body = params
+    else
+      req.set_form_data(params)
+    end
+    http.request(req)
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http-puma
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Masahiro Nakagawa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.50
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.50
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Puma based HTTP(S) plugin for Fluentd
+email: repeatedly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-http-puma.gemspec
+- lib/fluent/plugin/in_http_puma.rb
+- test/helper.rb
+- test/plugin/cert_puma.pem
+- test/plugin/puma_keypair.pem
+- test/plugin/test_in_http_puma.rb
+homepage: https://github.com/repeatedly/fluent-plugin-http-puma
+licenses:
+- Apache License (2.0)
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Puma based HTTP(S) plugin for Fluentd
+test_files:
+- test/helper.rb
+- test/plugin/cert_puma.pem
+- test/plugin/puma_keypair.pem
+- test/plugin/test_in_http_puma.rb