fluent-plugin-http-ex 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MGZlNmQ3NTVmNDkyNDRjNTMyYjYzMWFkNzJjYzBjZTdkYzNlZDI1Yw==
+  data.tar.gz: !binary |-
+    NTFkNjIyN2MyNDgwNmY2ZmRkMDNhMDAxYjZlNmU0ZmRmMTQ0ZmY4Ng==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MjFiNWVjNDI2YTYzNjNkZjZlY2Y5ZDJjMGQ5YzMyYjQ1ZmRjNTExZjBkYmNj
+    MGRmMDlhNmY3ZDc4ZjE5OTk1ZWVlZTg4MGM2MzBlNjNhZDMyOTQwNGFkZjVj
+    NTM5MWFkM2RiYjY5YzEyZTE0MDM2YzZjZWY5YzJiNDlkMTEyN2E=
+  data.tar.gz: !binary |-
+    MDUwZGEwN2YxZjcyMmMxYmUxYzk0YmJmNzRmOTk2ZDVlY2RiMDNlNjNmMGQz
+    NDlhZGRiNDNkMGJjZWRhNDlmNjZkMzg4YzMxNTVlNjQxNWY5MDg4ZjRmZTkx
+    NWUyNjJkODU1NmYyMGZhZGE3ZjIzNThkNGI1M2Q1NGMyMjM2NGI=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem "fluentd"
+
+group :test do
+  gem "nio4r"
+end

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (C) 2013 hiro-su
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,279 @@
+# Fluent::Plugin::Http::Ex
+
+## Overview
+
+This plugin takes JSON or MessagePack of events as input via a single, list or chunked
+HTTP POST request and emits each as an individual event to your output plugins.
+If you're sending a lot of events this simplifies your client's code and eliminates
+the overhead of creating a lot of brief connections.
+
+## Configuration
+
+The ExHttpInput plugin uses the same settings you would use for the standard
+HTTP input plugin. Example:
+
+    <source>
+      type http_ex
+      port 8888
+      bind 0.0.0.0
+      body_size_limit 32m
+      keepalive_timeout 300s #0s is not timeout
+    </source>
+
+Like the HTTP input plugin, the tag is determined by the URL used, which means 
+all events in one request must have the same tag.
+
+## Usage
+
+Have your logging system send JSON or Message of events. Example:
+
+Base URL
+
+    http://localhost:8888
+
+### json
+####  case 1
+
+resource
+
+    j or null
+
+header
+
+    Content-type: application/x-www-form-urlencoded
+
+body
+
+    json=<json data>
+
+sample
+
+    $ curl -X POST -d 'json={"action":"login","user":2}' \
+        http://localhost:8888/j/test.tag.here;
+
+    $ curl -X POST -d 'json={"action":"login","user":2}' \
+        http://localhost:8888/test.tag.here;
+
+#### case 2
+
+resource
+
+    j or null
+
+header
+
+    Content-type: application/json
+
+body
+
+    <json data>
+
+sample
+
+    $ curl -X POST -H 'Content-Type: application/json' -d '{"action":"login","user":2}' \
+        http://localhost:8888/j/test.tag.here;
+
+    $ curl -X POST -H 'Content-Type: application/json' -d '{"action":"login","user":2}' \
+        http://localhost:8888/test.tag.here;
+
+### json list
+#### case 1
+
+resource
+
+    js
+
+header
+
+    Content-type: application/x-www-form-urlencoded
+
+body
+
+    json=<json list data>
+
+sample
+
+    $ curl -X POST -d 'json=[{"action":"login","user":2},{"action":"login","user":2}]' \
+        http://localhost:8888/js/test.tag.here;
+
+#### case 2
+
+resource
+
+    js
+
+header
+
+    Content-type: application/json
+
+body
+
+    json=<json list data>
+
+sample
+
+    $ curl -X POST -d '[{"action":"login","user":2},{"action":"login","user":2}]' \
+        http://localhost:8888/js/test.tag.here;
+
+### msgpack
+#### case 1
+
+resource
+
+    m or null
+
+header
+
+    Content-type: application/x-www-form-urlencoded
+
+body
+
+    msgpack=<hash msgpack data>
+             hash.to_msgpack
+
+#### case2
+
+resource
+
+    m or null
+
+header
+
+    Content-type: application/x-msgpack
+
+body
+
+    <msgpack data>
+     hash.to_msgpack
+
+### msgpack list
+#### case 1
+
+resource
+
+    ms
+
+header
+
+    Content-type: application/x-www-form-urlencoded
+
+body
+
+    msgpack=<msgpack list data>
+             [hash,hash,hash].to_msgpack
+
+#### case 2
+
+resource
+
+    ms
+
+header
+
+    Content-type: application/x-msgpack
+
+body
+
+    msgpack=<msgpack list data>
+             [hash,hash,hash].to_msgpack
+
+### msgpack chunked
+
+resource
+
+    ms
+
+header
+
+    Content-type: application/x-msgpack
+    Transfer-Encoding: chunked
+
+body
+
+    <msgpack chunk data>
+     "#{hash.to_msgpack}#{hash.to_msgpack}"...
+
+
+Each event in the list will be sent to your output plugins as an individual
+event. 
+
+## Performance
+
+Comparison of in_http and in_http_ex.
+send 10,000 messages.
+
+
+machine spec
+
+    Mac OS X 10.8.2
+    1.8 GHz Intel Core i5
+    8 GB 1600 MHz DDR3
+
+### in_http
+
+json
+
+    $ time ruby examples/json.rb
+    
+    real    2m27.480s
+    user    0m7.252s
+    sys     0m4.438s
+
+msgpack
+
+    $ time ruby examples/msgpack.rb
+    
+    real    2m36.408s
+    user    0m8.249s
+    sys     0m4.441s
+
+### in_http_ex
+
+json
+
+    $ time ruby examples/json.rb
+    
+    real    2m30.639s
+    user    0m7.195s
+    sys     0m4.686s
+
+msgpack
+
+    $ time ruby examples/msgpack.rb
+    
+    real    2m28.442s
+    user    0m7.126s
+    sys     0m4.324s
+
+json list
+
+    $ time ruby examples/json_list.rb
+    
+    real    0m18.179s
+    user    0m0.872s
+    sys     0m0.477s
+
+msgpack list
+
+    $ time ruby examples/msgpack_list.rb
+    
+    real    0m13.787s
+    user    0m0.908s
+    sys     0m0.470s
+
+msgpack chunked
+
+    $ time ruby examples/nc_chunked.rb
+    
+    real    0m1.584s
+    user    0m0.244s
+    sys     0m0.107s
+
+
+## Copyright
+
+Copyright (c) 2013 hiro-su.
+
+Based on the in_http plugin by FURUHASHI Sadayuki
+
+ Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,10 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/examples/chunked.rb

@@ -0,0 +1,27 @@
+require 'net/http'
+require 'stringio'
+require 'msgpack'
+require 'json'
+
+host, port = 'localhost', 8888
+path = "/ms/test.tag"
+http = Net::HTTP.new(host, port)
+req = Net::HTTP::Post.new(path)
+req[ "Content-Type" ] = 'application/x-msgpack'
+req[ "Transfer-Encoding" ] = "chunked"
+req[ "Connection" ] = "Keep-Alive"
+
+io = StringIO.new
+DATA.each do |line|
+  io << JSON.parse(line.chomp).to_msgpack
+end
+io.rewind
+req.body_stream = io
+http.request(req).body
+
+__END__
+{"key1":"value1"}
+{"key2":"value2"}
+{"key3":"value3"}
+{"key4":"value4"}
+{"key5":"value5"}

data/examples/json.rb

@@ -0,0 +1,23 @@
+require 'net/http'
+
+# in_http
+#host, port = 'localhost', 7777
+#path = "/test.tag"
+#http = Net::HTTP.new(host, port)
+#req = Net::HTTP::Post.new(path)
+#req["Content-Type"] = "application/json"
+#1.upto(10000) do |i|
+#  req.body = "{\"key#{i}\":\"value#{i}\"}"
+#  http.request(req)
+#end
+
+# in_http_ex
+host, port = 'localhost', 8888
+path = "/j/test.tag"
+http = Net::HTTP.new(host, port)
+req = Net::HTTP::Post.new(path)
+req["Content-Type"] = "application/json"
+1.upto(10000) do |i|
+  req.body = "{\"key#{i}\":\"value#{i}\"}"
+  http.request(req)
+end

data/examples/json_list.rb

@@ -0,0 +1,12 @@
+require 'net/http'
+require 'msgpack'
+
+host, port = 'localhost', 8888
+path = "/js/test.tag"
+http = Net::HTTP.new(host, port)
+req = Net::HTTP::Post.new(path)
+req["Content-Type"] = "application/json"
+1.upto(1000) do |i|
+  req.body = "[{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"},{\"key#{i}\":\"value#{i}\"}]"
+  http.request(req)
+end

data/examples/msgpack.rb

@@ -0,0 +1,22 @@
+require 'net/http'
+require 'msgpack'
+
+# in_http
+#host, port = 'localhost', 7777
+#path = "/test.tag"
+#http = Net::HTTP.new(host, port)
+#1.upto(10000) do |i|
+#  record = URI.encode_www_form({"msgpack"=>{"key#{i}"=>"value#{i}"}.to_msgpack})
+#  http.post(path, record)
+#end
+
+# in_http_ex
+host, port = 'localhost', 8888
+path = "/m/test.tag"
+http = Net::HTTP.new(host, port)
+req = Net::HTTP::Post.new(path)
+req["Content-Type"] = "application/x-msgpack"
+1.upto(10000) do |i|
+  req.body = {"key#{i}"=>"value#{i}"}.to_msgpack
+  http.request(req)
+end

data/examples/msgpack_list.rb

@@ -0,0 +1,23 @@
+require 'net/http'
+require 'msgpack'
+
+host, port = 'localhost', 8888
+path = "/ms/test.tag"
+http = Net::HTTP.new(host, port)
+req = Net::HTTP::Post.new(path)
+req["Content-Type"] = "application/x-msgpack"
+1.upto(1000) do |i|
+  req.body = [
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+    {"key#{i}"=>"value#{i}"},
+  ].to_msgpack
+  http.request(req)
+end

data/examples/nc_chunked.rb

@@ -0,0 +1,44 @@
+require 'open3'
+require 'msgpack'
+
+class ChunkTest
+  def initialize(host, port)
+    @host = host
+    @port = port
+    @cmd = "nc"
+    @term = false
+  end
+
+  def run
+    Signal.trap(:INT){
+      @term = true
+    }
+    Open3.popen3("#{@cmd} #{@host} #{@port}") do |stdin, stdout, stderr, wait_thr|
+      begin
+        i = 0
+        loop do
+          break if @term
+          body = {"key#{i}"=>"value#{i}"}.to_msgpack
+          size = body.size
+          head = \
+            "POST /ms/test.tag HTTP/1.1\r\nUser-Agent: curl/7.28.0\r\nHost: #{@host}:#{@port}\r\nContent-type: application/x-msgpack\r\nTransfer-Encoding: chunked\r\nConnection: Keep-Alive\r\nExpect: 100-continue\r\n\r\n"
+          if i == 0
+            stdin << head
+          elsif i > 10000
+            stdin << "0\r\n\r\n"
+            break
+          else
+            chunk = "#{size.to_s(16)}\r\n#{body}\r\n"
+            stdin << chunk
+          end
+          i += 1
+        end
+      ensure
+        stdin.close
+      end
+    end
+  end
+end
+
+chunk = ChunkTest.new('localhost', 8888)
+chunk.run

data/examples/sample.conf

@@ -0,0 +1,11 @@
+<source>
+  type http_ex
+  port 8888
+  bind 0.0.0.0
+  body_size_limit 300M
+  keepalive_timeout 0s
+</source>
+
+<match **>
+  type stdout
+</match>

data/fluent-plugin-http-ex.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-http-ex"
+  spec.version       = "0.0.1"
+  spec.authors       = ["hiro-su"]
+  spec.email         = ["h.sugipon@gmail.com"]
+  spec.description   = %q{fluent plugin to accept multiple json/msgpack events in HTTP request}
+  spec.summary       = %q{fluent plugin to accept multiple json/msgpack events in HTTP request}
+  spec.homepage      = "https://github.com/hiro-su/fluent-plugin-http-ex"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "fluentd"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_http_ex.rb

@@ -0,0 +1,252 @@
+module Fluent
+
+class ExHttpInput < HttpInput
+  Plugin.register_input('http_ex', self)
+
+  config_param :port, :integer, :default => 8888
+  config_param :bind, :string, :default => '0.0.0.0'
+  config_param :body_size_limit, :size, :default => 32*1024*1024
+  config_param :keepalive_timeout, :time, :default => 30
+
+  class KeepaliveManager < Coolio::TimerWatcher
+    class TimerValue
+      def initialize
+        @value = 0
+      end
+      attr_accessor :value
+    end
+
+    def initialize(timeout)
+      super(1, true)
+      @cons = {}
+      @timeout = timeout.to_i
+    end
+
+    def add(sock)
+      @cons[sock] = sock
+    end
+
+    def delete(sock)
+      @cons.delete(sock)
+    end
+
+    def on_timer
+      @cons.each_pair {|sock,val|
+        if sock.step_idle > @timeout
+          sock.close unless @timeout == 0
+        end
+      }
+    end
+  end
+
+  def start
+    $log.debug "listening http on #{@bind}:#{@port}"
+    lsock = TCPServer.new(@bind, @port)
+
+    detach_multi_process do
+      Input.new.start
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      @lsock = Coolio::TCPServer.new(lsock, nil, ExHandler, @km, method(:on_request), @body_size_limit)
+
+      @loop = Coolio::Loop.new
+      @loop.attach(@km)
+      @loop.attach(@lsock)
+
+      @thread = Thread.new(&method(:run))
+    end
+  end
+  
+  def on_request(path_info, params)
+    $log.debug "remote_addr: #{params["REMOTE_ADDR"]}, path_info: #{path_info}"
+    begin
+      path = path_info[1..-1] # remove /
+      resource, tag = path.split('/')
+      tag ||= resource
+
+      if chunk = params['chunked']
+        record = chunk 
+        
+      elsif js = params['json']
+        record = JSON.parse(js)
+
+      elsif ms = params['x-msgpack'] || ms = params['msgpack']
+        record = MessagePack::unpack(ms)
+
+      else
+        raise "'json' or 'msgpack' parameter is required"
+      end
+
+      time = params['time']
+      time = time.to_i
+      if time == 0
+        time = Engine.now
+      end
+
+    rescue
+      return ["400 Bad Request", {'Content-type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
+    end
+
+    # TODO server error
+    begin
+      case params["resource"]
+      when :js
+        record.each do |v|
+          line = begin
+            JSON.parse(v)
+          rescue TypeError
+            v #hash
+          end
+          Engine.emit(tag, time, line)
+        end
+
+      when :ms
+        record.each {|line| Engine.emit(tag, time, line) }
+
+      else
+        Engine.emit(tag, time, record)
+      end
+
+    rescue
+      return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+    end
+
+    return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+  end
+
+  class ExHandler < Handler
+    def on_close
+      $log.debug "close #{@remote_addr}:#{@remote_port}"
+      super
+    end
+
+    def on_headers_complete(headers)
+      expect = nil
+      size = nil
+      if @parser.http_version == [1, 1]
+        @keep_alive = true
+      else
+        @keep_alive = false
+      end
+      @env = {}
+      headers.each_pair {|k,v|
+        @env["HTTP_#{k.gsub('-','_').upcase}"] = v
+        case k
+        when /Expect/i
+          expect = v
+        when /Content-Length/i
+          size = v.to_i
+        when /Content-Type/i
+          @content_type = v
+        when /Connection/i
+          if v =~ /close/i
+            @keep_alive = false
+          elsif v =~ /Keep-alive/i
+            @keep_alive = true
+          end
+        when /Transfer-Encoding/i
+          if v =~ /chunked/i
+            @chunked = true
+          end
+        end
+      }
+      if expect
+        if expect == '100-continue'
+          if !size || size < @body_size_limit
+            send_response_nobody("100 Continue", {})
+          else
+            send_response_and_close("413 Request Entity Too Large", {}, "Too large")
+          end
+        else
+          send_response_and_close("417 Expectation Failed", {}, "")
+        end
+      end
+    end
+
+    def on_body(chunk)
+      if @chunked && @content_type =~ /application\/x-msgpack/i
+        m = method(:on_read_msgpack)
+        @u = MessagePack::Unpacker.new
+        (class << self; self; end).module_eval do
+          define_method(:on_body, m)
+        end
+        m.call(chunk)
+      else
+        if @body.bytesize + chunk.bytesize > @body_size_limit
+          unless closing?
+            send_response_and_close("413 Request Entity Too Large", {}, "Too large")
+          end
+          return
+        end
+        @body << chunk
+      end
+    end
+
+    def on_read_msgpack(data)
+      params = WEBrick::HTTPUtils.parse_query(@parser.query_string)
+      path_info = @parser.request_path
+      @u.feed_each(data) do |obj|
+        params["chunked"] = obj
+        params["REMOTE_ADDR"] = @remote_addr
+        @callback.call(path_info, params)
+      end
+    rescue
+      $log.error "on_read_msgpack error: #{$!.to_s}"
+      $log.error_backtrace
+      close
+    end
+
+    def on_message_complete
+      return if closing?
+
+      @env['REMOTE_ADDR'] = @remote_addr
+
+      params = WEBrick::HTTPUtils.parse_query(@parser.query_string)
+      path_info = @parser.request_path
+
+      params = check_content_type(params, @content_type, @body, path_info)
+      params.merge!(@env)
+      @env.clear
+
+      unless @chunked
+        code, header, body = *@callback.call(path_info, params)
+        body = body.to_s
+
+        if @keep_alive
+          header['Connection'] = 'Keep-Alive'
+          send_response(code, header, body)
+        else
+          send_response_and_close(code, header, body)
+        end
+      else
+        send_response("200 OK", {'Content-type'=>'text/plain', 'Connection'=>'Keep-Alive'}, "")
+      end
+    end
+
+    def check_content_type(params, content_type, body, path_info)
+      path = path_info[1..-1] # remove /
+      resource, _ = path.split('/')
+      case resource
+      when /^js$/
+        params["resource"] = :js
+      when /^ms$/
+        params["resource"] = :ms
+      end
+
+      if content_type =~ /^application\/x-www-form-urlencoded/
+        params.update WEBrick::HTTPUtils.parse_query(body)
+      elsif content_type =~ /^multipart\/form-data; boundary=(.+)/
+        boundary = WEBrick::HTTPUtils.dequote($1)
+        params.update WEBrick::HTTPUtils.parse_form_data(body, boundary)
+      elsif content_type =~ /^application\/(json|x-msgpack)/
+        params[$1] = body
+      end
+
+      params
+    rescue => ex
+      $log.error ex
+      $log.error ex.backtrace * "\n"
+    end
+  end
+end
+
+end

data/test/helper.rb

@@ -0,0 +1,23 @@
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+require 'in_http_ex_test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_http'
+require 'fluent/plugin/in_http_ex'
+
+class Test::Unit::TestCase
+end

data/test/in_http_ex_test.rb

@@ -0,0 +1,100 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Test
+
+
+class ExHttpInputTestDriver < TestDriver
+  def initialize(klass, &block)
+    super(klass, &block)
+    @emit_streams = []
+    @expects = nil
+  end
+
+  def expect_emit(tag, time, record)
+    (@expects ||= []) << [tag, time, record]
+    self
+  end
+
+  def expected_emits
+    @expects ||= []
+  end
+
+  attr_reader :emit_streams
+
+  def emits
+    all = []
+    @emit_streams.each {|tag,events|
+      events.each {|time,record|
+        all << [tag, time, record]
+      }
+    }
+    all
+  end
+
+  def events
+    all = []
+    @emit_streams.each {|tag,events|
+      all.concat events
+    }
+    all
+  end
+
+  def records
+    all = []
+    @emit_streams.each {|tag,events|
+      events.each {|time,record|
+        all << record
+      }
+    }
+    all
+  end
+
+  def run(&block)
+    m = method(:emit_stream)
+    super {
+      Engine.define_singleton_method(:emit_stream) {|tag,es|
+        m.call(tag, es)
+      }
+
+      block.call if block
+
+      if @expects && @emit_streams.size < 2
+        i = 0
+        @emit_streams.each {|tag,events|
+          events.each {|time,record|
+            assert_equal(@expects[i], [tag, time, record])
+          }
+        }
+        assert_equal @expects.length, i
+      elsif @expects
+        return [@expects, @emit_streams]
+      end
+    }
+    self
+  end
+
+  private
+  def emit_stream(tag, es)
+    @emit_streams << [tag, es.to_a]
+  end
+end
+
+
+end
+end

data/test/plugin/test_in_http_ex.rb

@@ -0,0 +1,347 @@
+require 'helper'
+require 'net/http'
+
+class ExHttpInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+port 9911
+bind 127.0.0.1
+body_size_limit 10m
+keepalive_timeout 5
+]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::ExHttpInputTestDriver.new(Fluent::ExHttpInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 9911, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 10*1024*1024, d.instance.body_size_limit
+    assert_equal 5, d.instance.keepalive_timeout
+  end
+
+  def test_time
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}", {"json"=>record.to_json})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_application_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
+        req.body = record.to_json
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_resource_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/j/#{tag}", {"json"=>record.to_json, "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_application_resource_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/j/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
+        req.body = record.to_json
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_resource_json_list
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+    d.expect_emit "tag3", time, {"a"=>3}
+    
+    rs = d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/js/#{tag}", {"json"=>"[#{record.to_json},#{record.to_json},#{record.to_json}]", "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+
+    expects, results = rs
+    i, c = 0, 0
+    results.each {|tag,es|
+      es.each {|time,record|
+        assert_equal(expects[i], [tag, time, record])
+        c += 1
+        i += 1 if c % 3 == 0
+      }
+    }
+  end
+
+  def test_application_resource_json_list
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+    d.expect_emit "tag3", time, {"a"=>3}
+
+    rs = d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/js/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
+        req.body = "[#{record.to_json},#{record.to_json},#{record.to_json}]"
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+
+    expects, results = rs
+    i, c = 0, 0
+    results.each {|tag,es|
+      es.each {|time,record|
+        assert_equal(expects[i], [tag, time, record])
+        c += 1
+        i += 1 if c % 3 == 0
+      }
+    }
+  end
+
+  def test_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}", {"msgpack"=>record.to_msgpack, "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_application_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/#{tag}?time=#{time.to_s}", {"content-type"=>"application/x-msgpack; charset=utf-8"})
+        req.body = record.to_msgpack
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_resource_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/m/#{tag}", {"msgpack"=>record.to_msgpack, "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_application_resource_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/m/#{tag}?time=#{time.to_s}", {"content-type"=>"application/x-msgpack; charset=utf-8"})
+        req.body = record.to_msgpack
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_resource_msgpack_list
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+    d.expect_emit "tag3", time, {"a"=>3}
+    
+    rs = d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/ms/#{tag}", {"msgpack"=>[record,record,record].to_msgpack, "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+
+    expects, results = rs
+    i, c = 0, 0
+    results.each {|tag,es|
+      es.each {|time,record|
+        assert_equal(expects[i], [tag, time, record])
+        c += 1
+        i += 1 if c % 3 == 0
+      }
+    }
+  end
+
+  def test_application_resource_msgpack_list
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+    d.expect_emit "tag3", time, {"a"=>3}
+
+    rs = d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/ms/#{tag}?time=#{time.to_s}", {"content-type"=>"application/x-msgpack; charset=utf-8"})
+        req.body = [record,record,record].to_msgpack
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+
+    expects, results = rs
+    i, c = 0, 0
+    results.each {|tag,es|
+      es.each {|time,record|
+        assert_equal(expects[i], [tag, time, record])
+        c += 1
+        i += 1 if c % 3 == 0
+      }
+    }
+  end
+
+  def test_msgpack_chunked
+    require 'stringio'
+
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+    d.expect_emit "tag1", time, {"a"=>3}
+
+    http = Net::HTTP.new("127.0.0.1", 9911)
+
+    rs = d.run do
+      io = StringIO.new
+      req = ""
+      d.expected_emits.each {|tag,time,record|
+        req = Net::HTTP::Post.new("/ms/#{tag}?time=#{time.to_s}")
+        req["Content-Type"] = "application/x-msgpack"
+        req["Transfer-Encoding"] = "chunked"
+        io << record.to_msgpack
+        io << record.to_msgpack
+        io << record.to_msgpack
+      }
+      io.rewind
+      req.body_stream = io
+      res = http.request(req)
+      assert_equal "200", res.code
+    end
+
+    expects, results = rs
+    i, c = 0, 0
+    results.each {|tag,es|
+      es.each {|time,record|
+        assert_equal(expects[i], [tag, time, record])
+        c += 1
+        i += 1 if c % 3 == 0
+      }
+    }
+  end
+
+  def post(path, params)
+    http = Net::HTTP.new("127.0.0.1", 9911)
+    req = Net::HTTP::Post.new(path, {})
+    req.set_form_data(params)
+    http.request(req)
+  end
+
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http-ex
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- hiro-su
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluent plugin to accept multiple json/msgpack events in HTTP request
+email:
+- h.sugipon@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/chunked.rb
+- examples/json.rb
+- examples/json_list.rb
+- examples/msgpack.rb
+- examples/msgpack_list.rb
+- examples/nc_chunked.rb
+- examples/sample.conf
+- fluent-plugin-http-ex.gemspec
+- lib/fluent/plugin/in_http_ex.rb
+- test/helper.rb
+- test/in_http_ex_test.rb
+- test/plugin/test_in_http_ex.rb
+homepage: https://github.com/hiro-su/fluent-plugin-http-ex
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: fluent plugin to accept multiple json/msgpack events in HTTP request
+test_files:
+- test/helper.rb
+- test/in_http_ex_test.rb
+- test/plugin/test_in_http_ex.rb