fluentd 0.10.42 → 0.10.43

data/lib/fluent/plugin/out_copy.rb

@@ -37,7 +37,7 @@ module Fluent
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end
-        $log.debug "adding store type=#{type.dump}"
+        log.debug "adding store type=#{type.dump}"
 
         output = Plugin.new_output(type)
         output.configure(e)

data/lib/fluent/plugin/out_exec_filter.rb

@@ -114,7 +114,7 @@ module Fluent
           @time_format_proc = Proc.new {|time| time.to_s }
         end
       elsif @in_time_format
-        $log.warn "in_time_format effects nothing when in_time_key is not specified: #{conf}"
+        log.warn "in_time_format effects nothing when in_time_key is not specified: #{conf}"
       end
 
       if @out_time_key
@@ -124,7 +124,7 @@ module Fluent
           @time_parse_proc = Proc.new {|str| str.to_i }
         end
       elsif @out_time_format
-        $log.warn "out_time_format effects nothing when out_time_key is not specified: #{conf}"
+        log.warn "out_time_format effects nothing when out_time_key is not specified: #{conf}"
       end
 
       if @remove_prefix
@@ -180,7 +180,7 @@ module Fluent
       @rr = 0
       begin
         @num_children.times do
-          c = ChildProcess.new(@parser, @respawns)
+          c = ChildProcess.new(@parser, @respawns, log)
           c.start(@command)
           @children << c
         end
@@ -192,7 +192,7 @@ module Fluent
 
     def before_shutdown
       super
-      $log.debug "out_exec_filter#before_shutdown called"
+      log.debug "out_exec_filter#before_shutdown called"
       @children.each {|c|
         c.finished = true
       }
@@ -238,13 +238,14 @@ module Fluent
     class ChildProcess
       attr_accessor :finished
 
-      def initialize(parser,respawns=0)
+      def initialize(parser, respawns=0, log = $log)
         @pid = nil
         @thread = nil
         @parser = parser
         @respawns = respawns
         @mutex = Mutex.new
         @finished = nil
+        @log = log
       end
 
       def start(command)
@@ -289,7 +290,7 @@ module Fluent
           chunk.write_to(@io)
         rescue Errno::EPIPE => e
           # Broken pipe (child process unexpectedly exited)
-          $log.warn "exec_filter Broken pipe, child process maybe exited.", :command => @command
+          @log.warn "exec_filter Broken pipe, child process maybe exited.", :command => @command
           if try_respawn
             retry # retry chunk#write_to with child respawned
           else
@@ -312,21 +313,21 @@ module Fluent
 
           @respawns -= 1 if @respawns > 0
         end
-        $log.warn "exec_filter child process successfully respawned.", :command => @command, :respawns => @respawns
+        @log.warn "exec_filter child process successfully respawned.", :command => @command, :respawns => @respawns
         true
       end
 
       def run
         @parser.call(@io)
       rescue
-        $log.error "exec_filter thread unexpectedly failed with an error.", :command=>@command, :error=>$!.to_s
-        $log.warn_backtrace $!.backtrace
+        @log.error "exec_filter thread unexpectedly failed with an error.", :command=>@command, :error=>$!.to_s
+        @log.warn_backtrace $!.backtrace
       ensure
         pid, stat = Process.waitpid2(@pid)
         unless @finished
-          $log.error "exec_filter process unexpectedly exited.", :command=>@command, :ecode=>stat.to_i
+          @log.error "exec_filter process unexpectedly exited.", :command=>@command, :ecode=>stat.to_i
           unless @respawns == 0
-            $log.warn "exec_filter child process will respawn for next input data (respawns #{@respawns})."
+            @log.warn "exec_filter child process will respawn for next input data (respawns #{@respawns})."
           end
         end
       end
@@ -385,8 +386,8 @@ module Fluent
 
     rescue
       if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
-        $log.error "exec_filter failed to emit", :error=>$!.to_s, :error_class=>$!.class.to_s, :record=>Yajl.dump(record)
-        $log.warn_backtrace $!.backtrace
+        log.error "exec_filter failed to emit", :error=>$!.to_s, :error_class=>$!.class.to_s, :record=>Yajl.dump(record)
+        log.warn_backtrace $!.backtrace
         @next_log_time = Time.now.to_i + @suppress_error_log_interval
       end
     end

data/lib/fluent/plugin/out_forward.rb

@@ -54,7 +54,7 @@ module Fluent
 
       # backward compatibility
       if host = conf['host']
-        $log.warn "'host' option in forward output is obsoleted. Use '<server> host xxx </server>' instead."
+        log.warn "'host' option in forward output is obsoleted. Use '<server> host xxx </server>' instead."
         port = conf['port']
         port = port ? port.to_i : DEFAULT_LISTEN_PORT
         e = conf.add_element('server')
@@ -83,8 +83,8 @@ module Fluent
 
         failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
         @nodes << Node.new(name, host, port, weight, standby, failure,
-          @phi_threshold, recover_sample_size, @expire_dns_cache)
-        $log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight
+          @phi_threshold, recover_sample_size, @expire_dns_cache, log)
+        log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight
       }
     end
 
@@ -122,8 +122,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     def write_objects(tag, chunk)
@@ -167,13 +167,13 @@ module Fluent
           lost_weight += n.weight
         end
       }
-      $log.debug "rebuilding weight array", :lost_weight=>lost_weight
+      log.debug "rebuilding weight array", :lost_weight=>lost_weight
 
       if lost_weight > 0
         standby_nodes.each {|n|
           if n.available?
             regular_nodes << n
-            $log.warn "using standby node #{n.host}:#{n.port}", :weight=>n.weight
+            log.warn "using standby node #{n.host}:#{n.port}", :weight=>n.weight
             lost_weight -= n.weight
             break if lost_weight <= 0
           end
@@ -279,7 +279,7 @@ module Fluent
           rebuild_weight_array
         end
         begin
-          #$log.trace "sending heartbeat #{n.host}:#{n.port} on #{@heartbeat_type}"
+          #log.trace "sending heartbeat #{n.host}:#{n.port} on #{@heartbeat_type}"
           if @heartbeat_type == :tcp
             send_heartbeat_tcp(n)
           else
@@ -287,7 +287,7 @@ module Fluent
           end
         rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
           # TODO log
-          $log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", :error=>$!.to_s
+          log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", :error=>$!.to_s
         end
       }
     end
@@ -317,7 +317,7 @@ module Fluent
     def on_heartbeat(sockaddr, msg)
       port, host = Socket.unpack_sockaddr_in(sockaddr)
       if node = @nodes.find {|n| n.sockaddr == sockaddr }
-        #$log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
+        #log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
         if node.heartbeat
           rebuild_weight_array
         end
@@ -326,7 +326,7 @@ module Fluent
 
     class Node
       def initialize(name, host, port, weight, standby, failure,
-          phi_threshold, recover_sample_size, expire_dns_cache)
+          phi_threshold, recover_sample_size, expire_dns_cache, log)
         @name = name
         @host = host
         @port = port
@@ -337,6 +337,7 @@ module Fluent
         @recover_sample_size = recover_sample_size
         @expire_dns_cache = expire_dns_cache
         @available = true
+        @log = log
 
         @resolved_host = nil
         @resolved_time = 0
@@ -393,7 +394,7 @@ module Fluent
         end
 
         if @failure.hard_timeout?(now)
-          $log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :hard_timeout=>true
+          @log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :hard_timeout=>true
           @available = false
           @resolved_host = nil  # expire cached host
           @failure.clear
@@ -403,7 +404,7 @@ module Fluent
         phi = @failure.phi(now)
         #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
         if phi > @phi_threshold
-          $log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
+          @log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
           @available = false
           @resolved_host = nil  # expire cached host
           @failure.clear
@@ -416,10 +417,10 @@ module Fluent
       def heartbeat(detect=true)
         now = Time.now.to_f
         @failure.add(now)
-        #$log.trace "heartbeat from '#{@name}'", :host=>@host, :port=>@port, :available=>@available, :sample_size=>@failure.sample_size
+        #@log.trace "heartbeat from '#{@name}'", :host=>@host, :port=>@port, :available=>@available, :sample_size=>@failure.sample_size
         if detect && !@available && @failure.sample_size > @recover_sample_size
           @available = true
-          $log.warn "recovered forwarding server '#{@name}'", :host=>@host, :port=>@port
+          @log.warn "recovered forwarding server '#{@name}'", :host=>@host, :port=>@port
           return true
         else
           return nil

data/lib/fluent/plugin/out_roundrobin.rb

@@ -20,6 +20,8 @@ module Fluent
     Plugin.register_output('roundrobin', self)
 
     def initialize
+      super
+
       @outputs = []
       @weights = []
     end
@@ -28,6 +30,8 @@ module Fluent
     attr_accessor :rand_seed
 
     def configure(conf)
+      super
+
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
@@ -38,7 +42,7 @@ module Fluent
 
         weight = e['weight']
         weight = weight ? weight.to_i : 1
-        $log.debug "adding store type=#{type.dump}, weight=#{weight}"
+        log.debug "adding store type=#{type.dump}, weight=#{weight}"
 
         output = Plugin.new_output(type)
         output.configure(e)

data/lib/fluent/plugin/out_stdout.rb

@@ -34,6 +34,7 @@ module Fluent
         raise ConfigError, "stdout output output_type should be 'json' or 'hash'"
       end
     end
+
     def configure(conf)
       super
       @output_proc = OUTPUT_PROCS[@output_type]

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.42'
+  VERSION = '0.10.43'
 
 end

data/test/output.rb

@@ -51,5 +51,66 @@ module FluentOutputTest
       d.instance.retry_limit.times { d.instance.instance_variable_get(:@error_history) << Engine.now }
       assert_equal 4, d.instance.calc_retry_wait
     end
+
+    def create_mock_driver(conf=CONFIG)
+      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+        attr_accessor :submit_flush_threads
+
+        def start_mock
+          @started = false
+          start
+          # ensure OutputThread to start successfully
+          submit_flush
+          sleep 0.5
+          while !@started
+            submit_flush
+            sleep 0.5
+          end
+        end
+
+        def try_flush
+          @started = true
+          @submit_flush_threads ||= {}
+          @submit_flush_threads[Thread.current] ||= 0
+          @submit_flush_threads[Thread.current] += 1
+        end
+
+        def write(chunk)
+          chunk.read
+        end
+      end.configure(conf)
+    end
+
+    def test_submit_flush_target
+      # default
+      d = create_mock_driver
+      d.instance.start_mock
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.shutdown
+      assert_equal 1, d.instance.submit_flush_threads.size
+
+      # num_threads 4
+      d = create_mock_driver(CONFIG + %[num_threads 4])
+      d.instance.start
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 1, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 2, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 3, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.submit_flush
+      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
+      d.instance.shutdown
+      assert (d.instance.submit_flush_threads.size > 1), "fails if only one thread works to submit flush"
+    end
   end
 end

data/test/parser.rb

@@ -64,6 +64,13 @@ module ParserTest
       internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
     end
 
+    def test_call_with_configure
+      # Specify conf by configure method instaed of intializer
+      parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!))
+      parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
+      internal_test_case(parser)
+    end
+
     def test_call_with_typed_and_name_separator
       internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
     end

data/test/plugin/in_http.rb

@@ -25,6 +25,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal '127.0.0.1', d.instance.bind
     assert_equal 10*1024*1024, d.instance.body_size_limit
     assert_equal 5, d.instance.keepalive_timeout
+    assert_equal false, d.instance.add_http_headers
   end
 
   def test_time
@@ -58,6 +59,30 @@ class HttpInputTest < Test::Unit::TestCase
         assert_equal "200", res.code
       }
     end
+
+    d.emit_streams.each { |tag, es|
+      assert !include_http_header?(es.first[1])
+    }
+  end
+
+  def test_json_with_add_http_headers
+    d = create_driver(CONFIG + "add_http_headers true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [["tag1", time, {"a"=>1}], ["tag2", time, {"a"=>2}]]
+
+    d.run do
+      records.each {|tag,time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time.to_s})
+        assert_equal "200", res.code
+
+      }
+    end
+
+    d.emit_streams.each { |tag, es|
+      assert include_http_header?(es.first[1])
+    }
   end
 
   def test_application_json
@@ -101,5 +126,8 @@ class HttpInputTest < Test::Unit::TestCase
     req.set_form_data(params)
     http.request(req)
   end
-end
 
+  def include_http_header?(record)
+    record.keys.find { |header| header.start_with?('HTTP_') }
+  end
+end

metadata

@@ -1,32 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.42
+  version: 0.10.43
+  prerelease: 
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-25 00:00:00.000000000 Z
+date: 2014-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -35,20 +37,21 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -57,20 +60,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.4.3
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -78,6 +84,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -85,6 +92,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: cool.io
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -92,6 +100,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -99,6 +108,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -106,64 +116,89 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: sigdump
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.15.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.15.3
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.3.0
 description: Fluentd is an event collector system. It is a generalized version of
@@ -271,26 +306,30 @@ files:
 - test/scripts/exec_script.rb
 homepage: http://fluentd.org/
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -936965180993264578
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Fluentd event collector
 test_files:
 - test/config.rb