fluent-plugin-http-list 0.1.0

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+test.conf
+.idea

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+#
+# Fluent HTTP List plugin
+#
+# Copyright 2013 M3, Inc.
+# Written by Paul McCann (p-mccann@m3.com)
+# Based on the HTTP Input Plugin by FURUHASHI Sadayuki included with Fluentd
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+##
+

data/README.md

@@ -0,0 +1,42 @@
+# fluent-plugin-http-list
+
+## Overview
+
+This plugin takes a JSON list of events as input via HTTP POST. If you're
+sending a lot of events this simplifies your client's code and eliminates 
+the overhead of creating a lot of brief connections.
+
+※ Note that unlike the default HTTP plugin, this does *not* support msgpack. 
+
+## Configuration
+
+The HttpListInput plugin uses the same settings you would use for the standard
+HTTP input plugin. Example:
+
+    <source>
+      type http_list
+      port 8888
+      bind 0.0.0.0
+      body_size_limit 32m
+      keepalive_timeout 10s
+    </source>
+
+Like the HTTP input plugin, the tag is determined by the URL used, which means 
+all events in one request must have the same tag.
+
+## Usage
+
+Have your logging system send JSON lists of events. Example:
+
+    curl -X POST -d 'json=[{"fish":"catfish","user":23},{"fish":"elephantfish","user":23}]' \
+      http://localhost:8888/fish.tracker
+
+Each event will go to your output plugins as an individual event. 
+
+## Copyright
+
+Copyright (c) 2013 M3, Inc. 
+
+Based on the in_http plugin by FURUHASHI Sadayuki
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-http-list.gemspec

@@ -0,0 +1,18 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-http-list"
+  gem.version       = "0.1.0"
+  gem.authors       = ["Paul McCann"]
+  gem.email         = ["polm@dampfkraft.com"]
+  gem.description   = %q{fluent plugin to accept multiple events in one HTTP request}
+  gem.summary       = %q{fluent plugin to accept multiple events in one HTTP request}
+  gem.homepage      = "https://github.com/m3dev/fluent-plugin-http-list"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_http_list.rb

@@ -0,0 +1,269 @@
+module Fluent
+
+class HttpListInput < Input
+  Plugin.register_input('http_list', self)
+
+  include DetachMultiProcessMixin
+
+  require 'http/parser'
+
+  def initialize
+    require 'webrick/httputils'
+    super
+  end
+
+  config_param :port, :integer, :default => 9880
+  config_param :bind, :string, :default => '0.0.0.0'
+  config_param :body_size_limit, :size, :default => 32*1024*1024 
+  config_param :keepalive_timeout, :time, :default => 10   
+
+  def configure(conf)
+    super
+  end
+
+  class KeepaliveManager < Coolio::TimerWatcher
+    class TimerValue
+      def initialize
+        @value = 0
+      end
+      attr_accessor :value
+    end
+
+    def initialize(timeout)
+      super(1, true)
+      @cons = {}
+      @timeout = timeout.to_i
+    end
+
+    def add(sock)
+      @cons[sock] = sock
+    end
+
+    def delete(sock)
+      @cons.delete(sock)
+    end
+
+    def on_timer
+      @cons.each_pair {|sock,val|
+        if sock.step_idle > @timeout
+          sock.close
+        end
+      }
+    end
+  end
+
+  def start
+    $log.debug "listening for http on #{@bind}:#{@port}"
+    lsock = TCPServer.new(@bind, @port)
+
+    detach_multi_process do
+      super
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit)
+
+      @loop = Coolio::Loop.new
+      @loop.attach(@km)
+      @loop.attach(@lsock)
+
+      @thread = Thread.new(&method(:run))
+    end
+  end
+
+  def shutdown
+    @loop.watchers.each {|w| w.detach }
+    @loop.stop
+    @lsock.close
+    @thread.join
+  end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  def on_request(path_info, params)
+    begin
+      path = path_info[1..-1]  # remove /
+      tag = path.split('/').join('.')
+
+      if js = params['json']
+        records = JSON.parse(js)
+        p records
+      else
+        raise "'json' parameter is required" + params.keys.to_s
+      end
+
+    time = params['time'].nil? ? Engine.now : params['time'].to_i
+
+    rescue
+      return ["400 Bad Request", {'Content-type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
+    end
+
+
+    begin
+      records.each{|r| 
+          Engine.emit(tag, time, r)
+      }
+    rescue
+      return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+    end
+
+    return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+  end
+
+  class Handler < Coolio::Socket
+    def initialize(io, km, callback, body_size_limit)
+      super(io)
+      @km = km
+      @callback = callback
+      @body_size_limit = body_size_limit
+      @content_type = ""
+      @next_close = false
+
+      @idle = 0
+      @km.add(self)
+    end
+
+    def step_idle
+      @idle += 1
+    end
+
+    def on_close
+      @km.delete(self)
+    end
+
+    def on_connect
+      @parser = Http::Parser.new(self)
+    end
+
+    def on_read(data)
+      @idle = 0
+      @parser << data
+    rescue
+      $log.warn "unexpected error", :error=>$!.to_s
+      $log.warn_backtrace
+      close
+    end
+
+    def on_message_begin
+      @body = ''
+    end
+
+    def on_headers_complete(headers)
+      expect = nil
+      size = nil
+      if @parser.http_version == [1, 1]
+        #Modified to always be false - Paul McCann 2012/10/24
+        #Changed because it was likely cause of slowness in production
+        @keep_alive = false
+      else
+        @keep_alive = false
+      end
+      headers.each_pair {|k,v|
+        case k
+        when /Expect/i
+          expect = v
+        when /Content-Length/i
+          size = v.to_i
+        when /Content-Type/i
+          @content_type = v
+        when /Connection/i
+          if v =~ /close/i
+            @keep_alive = false
+          elsif v =~ /Keep-alive/i
+            @keep_alive = true
+          end
+        end
+      }
+      if expect
+        if expect == '100-continue'
+          if !size || size < @body_size_limit
+            send_response_nobody("100 Continue", {})
+          else
+            send_response_and_close("413 Request Entity Too Large", {}, "Too large")
+          end
+        else
+          send_response_and_close("417 Expectation Failed", {}, "")
+        end
+      end
+    end
+
+    def on_body(chunk)
+      if @body.bytesize + chunk.bytesize > @body_size_limit
+        unless closing?
+          send_response_and_close("413 Request Entity Too Large", {}, "Too large")
+        end
+        return
+      end
+      @body << chunk
+    end
+
+    def on_message_complete
+      return if closing?
+
+      params = WEBrick::HTTPUtils.parse_query(@parser.query_string)
+
+      if @content_type =~ /^application\/x-www-form-urlencoded/
+        params.update WEBrick::HTTPUtils.parse_query(@body)
+      elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
+        boundary = WEBrick::HTTPUtils.dequote($1)
+        params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
+      elsif @content_type =~ /^application\/json/
+        params['json'] = @body
+      end
+      path_info = @parser.request_path
+
+      code, header, body = *@callback.call(path_info, params)
+      body = body.to_s
+
+      if @keep_alive
+        header['Connection'] = 'Keep-Alive'
+        send_response(code, header, body)
+      else
+        send_response_and_close(code, header, body)
+      end
+    end
+
+    def on_write_complete
+      close if @next_close
+    end
+
+    def send_response_and_close(code, header, body)
+      send_response(code, header, body)
+      @next_close = true
+    end
+
+    def closing?
+      @next_close
+    end
+
+    def send_response(code, header, body)
+      header['Content-length'] ||= body.bytesize
+      header['Content-type'] ||= 'text/plain'
+
+      data = %[HTTP/1.1 #{code}\r\n]
+      header.each_pair {|k,v|
+        data << "#{k}: #{v}\r\n"
+      }
+      data << "\r\n"
+      write data
+
+      write body
+    end
+
+    def send_response_nobody(code, header)
+      data = %[HTTP/1.1 #{code}\r\n]
+      header.each_pair {|k,v|
+        data << "#{k}: #{v}\r\n"
+      }
+      data << "\r\n"
+      write data
+    end
+  end
+end
+
+
+end
+

data/test/plugin/test_in_http_list.rb

@@ -0,0 +1,156 @@
+require 'fluent/test'
+require 'net/http'
+
+class HttpListInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/in_http_list'
+  end
+
+  CONFIG = %[
+    port 9911
+    bind 127.0.0.1
+    body_size_limit 10m
+    keepalive_timeout 5
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::HttpListInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 9911, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 10*1024*1024, d.instance.body_size_limit
+    assert_equal 5, d.instance.keepalive_timeout
+  end
+
+  #
+  # sending events as a post parameter named 'json'
+  # e.g. curl -XPOST 'http://127.0.0.1:9911/tag1' -d'time=1357860203&json=[{"a":1}]'
+  #
+  def test_parameter
+    d = create_driver
+
+    time = Time.parse("2013-01-10 23:23:23 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post_as_parameter("/#{tag}", time.to_s, [record].to_json)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  #
+  # sending evens as a request body (application/json)
+  # e.g. curl -XPOST 'http://127.0.0.1:9911/tag1?time=1357860203' -d'[{"a":1}]' -H 'Content-Type:application/json; charset=utf-8'
+  #
+  def test_application_json
+    d = create_driver
+
+    time = Time.parse("2013-01-10 23:23:23 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post_as_application_json("/#{tag}", time.to_s, [record].to_json)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_multiple_events_as_parameter
+    d = create_driver
+
+    time = Time.parse("2013-01-10 23:23:23 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+    d.expect_emit "tag1", time, {"a"=>3}
+    d.expect_emit "tag2", time, {"a"=>4}
+    d.expect_emit "tag2", time, {"a"=>5}
+    d.expect_emit "tag2", time, {"a"=>6}
+
+    test_events = [
+        ["tag1", (1..3).to_a.collect {|x| {"a"=>x}}],
+        ["tag2", (4..6).to_a.collect {|x| {"a"=>x}}]
+    ]
+
+    d.run do
+      test_events.each {|tag, events|
+        res = post_as_parameter("/#{tag}", time, events.to_json)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_multiple_events_as_application_json
+    d = create_driver
+
+    time = Time.parse("2013-01-10 23:23:23 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+    d.expect_emit "tag1", time, {"a"=>3}
+    d.expect_emit "tag2", time, {"a"=>4}
+    d.expect_emit "tag2", time, {"a"=>5}
+    d.expect_emit "tag2", time, {"a"=>6}
+
+    test_events = [
+        ["tag1", (1..3).to_a.collect {|x| {"a"=>x}}],
+        ["tag2", (4..6).to_a.collect {|x| {"a"=>x}}]
+    ]
+
+    d.run do
+      test_events.each {|tag, events|
+        res = post_as_application_json("/#{tag}", time.to_s, events.to_json)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_zero_events_as_parameter
+    # An empty event list should not fail
+    d = create_driver
+    time = Time.parse("2013-01-10 23:23:23 UTC").to_i
+    d.run do
+      res = post_as_parameter("/zero", time.to_s, [].to_json)
+      assert_equal "200", res.code
+    end
+  end
+
+  def test_zero_events_as_application_json
+    # An empty event list should not fail
+    d = create_driver
+    time = Time.parse("2013-01-10 23:23:23 UTC").to_i
+    d.run do
+      res = post_as_application_json("/zero", time.to_s, [].to_json)
+      assert_equal "200", res.code
+    end
+  end
+
+  private
+
+  def post_as_parameter(path, time, json)
+    http = Net::HTTP.new("127.0.0.1", 9911)
+    req = Net::HTTP::Post.new(path, {})
+    req.set_form_data({"time" => time, "json" => json})
+    http.request(req)
+  end
+
+  def post_as_application_json(path, time, json)
+    http = Net::HTTP.new("127.0.0.1", 9911)
+    req = Net::HTTP::Post.new("#{path}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
+    req.body = json
+    http.request(req)
+  end
+
+end
+

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http-list
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Paul McCann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluent plugin to accept multiple events in one HTTP request
+email:
+- polm@dampfkraft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-http-list.gemspec
+- lib/fluent/plugin/in_http_list.rb
+- test/plugin/test_in_http_list.rb
+homepage: https://github.com/m3dev/fluent-plugin-http-list
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: fluent plugin to accept multiple events in one HTTP request
+test_files:
+- test/plugin/test_in_http_list.rb