fluentd 0.10.58 → 0.10.59

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a62e7c7181c7f247c4a76b42638d0a6b4195d8a
-  data.tar.gz: 12ccff6793701f59c0b67c17d66804a3d25ac818
+  metadata.gz: 434f3deef039256104205794d09c06668a82e8c4
+  data.tar.gz: 296981ab483797547447a60ce944987be7ff7431
 SHA512:
-  metadata.gz: 81a1d182345423ecbb41a94351ccb6666f449126128718187345b9e62c187a3f8ed50fa56cfdee19cd5817ed37620471d10d871ca339cd345ef2111e1da76e29
-  data.tar.gz: 7a61d7bd4b80b28373fb1ed7824add416a085d77a7ace5a229bb34ebb6965fadf7fbcbd5ff4d61ee89ce6166b05a17011b619d5e7d9e9e42ddade2d6828ab8f0
+  metadata.gz: 1f5cd6aef58345917d17e5ab6560d610954eba3f3043cad59d9a446f5b4aa4964e2d118e94a9b1be31610558fba1cccb74430fda55eb9e1e8744b6afee865af8
+  data.tar.gz: 2e6a4d27834b6cff8f8b7efe1cb1da480f935dd2e958f78aec5ded158e490216e311bcfc40437cf4caeba01248b88c4be4825dfd8f1ca36ba5673a4079f2209f

data/.gitignore

@@ -19,3 +19,4 @@ pkg/*
 test/tmp/*
 test/config/tmp/*
 make_dist.sh
+Gemfile.local

data/ChangeLog

@@ -1,3 +1,15 @@
+Release 0.10.59 - 2014/01/23
+
+* in_tail: Support 64bit inode environment in in_tail
+* parser: nginx and apache2 format can now parse access log without http-version
+* buffer: Fix broken calc_retry_wait if Integer is used for retry_wait parameter
+* buffer: Fix to flush a buffer by USR1 signal even on retrying
+* buffer: Fix TimeSlicedOutput doesn't flush with SIGUSR1
+* config: Support @ prefix build-in parameters
+* engine:Fix ThreadError occuring on Signal.trap SIGHUP
+* engine: Fix SIGHUP does not reload config
+* Revert fluent.conf since @type, @id are not available with v0 config
+
 Release 0.10.58 - 2014/12/14
 
 * parser/formatter: Add base class and Plugin.new_xxx/Plugin.register_xxx APIs

data/Gemfile

@@ -1,3 +1,9 @@
 source 'https://rubygems.org/'
 
 gemspec
+
+local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
+if File.exist?(local_gemfile)
+  puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v`
+  instance_eval File.read(local_gemfile)
+end

data/fluent.conf

@@ -4,20 +4,20 @@
 ## built-in TCP input
 ## $ echo <json> | fluent-cat <tag>
 <source>
-  @type forward
-  @id forward_input
+  type forward
+  id forward_input
 </source>
 
 ## built-in UNIX socket input
 #<source>
-#  @type unix
+#  type unix
 #</source>
 
 # HTTP input
 # http://localhost:8888/<tag>?json=<json>
 <source>
-  @type http
-  @id http_input
+  type http
+  id http_input
 
   port 8888
 </source>
@@ -25,7 +25,7 @@
 ## File input
 ## read apache logs with tag=apache.access
 #<source>
-#  @type tail
+#  type tail
 #  format apache
 #  path /var/log/httpd-access.log
 #  tag apache.access
@@ -36,16 +36,16 @@
 # http://localhost:24220/api/plugins?type=TYPE
 # http://localhost:24220/api/plugins?tag=MYTAG
 <source>
-  @type monitor_agent
-  @id monitor_agent_input
+  type monitor_agent
+  id monitor_agent_input
 
   port 24220
 </source>
 
 # Listen DRb for debug
 <source>
-  @type debug_agent
-  @id debug_agent_input
+  type debug_agent
+  id debug_agent_input
 
   bind 127.0.0.1
   port 24230
@@ -53,20 +53,20 @@
 
 ## match tag=apache.access and write to file
 #<match apache.access>
-#  @type file
+#  type file
 #  path /var/log/fluent/access
 #</match>
 
 ## match tag=debug.** and dump to console
 <match debug.**>
-  @type stdout
-  @id stdout_output
+  type stdout
+  id stdout_output
 </match>
 
 # match tag=system.** and forward to another fluent server
 <match system.**>
-  @type forward
-  @id forward_output
+  type forward
+  id forward_output
 
   <server>
     host 192.168.0.11
@@ -80,9 +80,9 @@
 
 ## match tag=myapp.** and forward and write to file
 #<match myapp.**>
-#  @type copy
+#  type copy
 #  <store>
-#    @type forward
+#    type forward
 #    buffer_type file
 #    buffer_path /var/log/fluent/myapp-forward
 #    retry_limit 50
@@ -92,19 +92,19 @@
 #    </server>
 #  </store>
 #  <store>
-#    @type file
+#    type file
 #    path /var/log/fluent/myapp
 #  </store>
 #</match>
 
 ## match fluent's internal events
 #<match fluent.**>
-#  @type null
+#  type null
 #</match>
 
 ## match not matched logs and write to file
 #<match **>
-#  @type file
+#  type file
 #  path /var/log/fluent/else
 #  compress gz
 #</match>

data/lib/fluent/config/v1_parser.rb

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   module Config
 
@@ -50,7 +51,7 @@ module Fluent
       end
 
       ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
-      RESERVED_PARAMS = %W(@type @id @label)
+      RESERVED_PARAMS = %W(@type @id)
 
       def parse_element(root_element, elem_name, attrs = {}, elems = [])
         while true
@@ -108,6 +109,12 @@ module Fluent
             else
               if k == '@include'
                 parse_include(attrs, elems)
+              elsif RESERVED_PARAMS.include?(k)
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
               else
                 if k.start_with?('@')
                   if root_element || ELEM_SYMBOLS.include?(elem_name)

data/lib/fluent/engine.rb

@@ -100,7 +100,7 @@ module Fluent
         conf.elements.select {|e|
           e.name == 'source'
         }.each {|e|
-          type = e['type']
+          type = e['@type'] || e['type']
           unless type
             raise ConfigError, "Missing 'type' parameter on <source> directive"
           end
@@ -116,7 +116,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'match'
       }.each {|e|
-        type = e['type']
+        type = e['@type'] || e['type']
         pattern = e.arg
         unless type
           raise ConfigError, "Missing 'type' parameter on <match #{e.arg}> directive"

data/lib/fluent/output.rb

@@ -188,6 +188,7 @@ module Fluent
     def configure(conf)
       super
 
+      @retry_wait = @retry_wait.to_f # converted to Float for calc_retry_wait
       @buffer = Plugin.new_buffer(@buffer_type)
       @buffer.configure(conf)
 
@@ -267,7 +268,7 @@ module Fluent
     #def write(chunk)
     #end
 
-    def enqueue_buffer
+    def enqueue_buffer(force = false)
       @buffer.keys.each {|key|
         @buffer.push(key)
       }
@@ -378,7 +379,10 @@ module Fluent
     end
 
     def force_flush
-      enqueue_buffer
+      @num_errors_lock.synchronize do
+        @next_retry_time = Engine.now - 1
+      end
+      enqueue_buffer(true)
       submit_flush
     end
 
@@ -546,8 +550,14 @@ module Fluent
       }
     end
 
-    def enqueue_buffer
-      @enqueue_buffer_proc.call
+    def enqueue_buffer(force = false)
+      if force
+        @buffer.keys.each {|key|
+          @buffer.push(key)
+        }
+      else
+        @enqueue_buffer_proc.call
+      end
     end
 
     #def format(tag, event)

data/lib/fluent/parser.rb

@@ -408,7 +408,7 @@ module Fluent
     end
 
     class ApacheParser < Parser
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
@@ -632,7 +632,7 @@ module Fluent
       'tsv' => Proc.new { TSVParser.new },
       'ltsv' => Proc.new { LabeledTSVParser.new },
       'csv' => Proc.new { CSVParser.new },
-      'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+      'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
       'none' => Proc.new { NoneParser.new },
       'multiline' => Proc.new { MultilineParser.new },
     }.each { |name, factory|

data/lib/fluent/plugin/buf_file.rb

@@ -84,6 +84,7 @@ module Fluent
     end
 
     config_param :buffer_path, :string
+    config_param :flush_at_shutdown, :bool, :default => false
 
     attr_accessor :symlink_path
 
@@ -104,11 +105,6 @@ module Fluent
         @buffer_path_suffix = ".log"
       end
 
-      if flush_at_shutdown = conf['flush_at_shutdown']
-        @flush_at_shutdown = true
-      else
-        @flush_at_shutdown = false
-      end
     end
 
     def start

data/lib/fluent/plugin/in_tail.rb

@@ -598,7 +598,7 @@ module Fluent
         @file.write path
         @file.write "\t"
         seek = @file.pos
-        @file.write "0000000000000000\t00000000\n"
+        @file.write "0000000000000000\t0000000000000000\n"
         @last_pos = @file.pos
 
         @map[path] = FilePositionEntry.new(@file, seek)
@@ -624,12 +624,15 @@ module Fluent
       # Clean up unwatched file entries
       def self.compact(file)
         file.pos = 0
-        existent_entries = file.each_line.select { |line|
+        existent_entries = file.each_line.map { |line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
           next unless m
+          path = m[1]
           pos = m[2].to_i(16)
-          pos == UNWATCHED_POSITION ? nil : line
-        }
+          ino = m[3].to_i(16)
+          # 32bit inode converted to 64bit at this phase
+          pos == UNWATCHED_POSITION ? nil : ("%s\t%016x\t%016x\n" % [path, pos, ino])
+        }.compact
 
         file.pos = 0
         file.truncate(0)
@@ -638,13 +641,13 @@ module Fluent
     end
 
     # pos               inode
-    # ffffffffffffffff\tffffffff\n
+    # ffffffffffffffff\tffffffffffffffff\n
     class FilePositionEntry
       POS_SIZE = 16
       INO_OFFSET = 17
-      INO_SIZE = 8
-      LN_OFFSET = 25
-      SIZE = 26
+      INO_SIZE = 16
+      LN_OFFSET = 33
+      SIZE = 34
 
       def initialize(file, seek)
         @file = file
@@ -653,7 +656,7 @@ module Fluent
 
       def update(ino, pos)
         @file.pos = @seek
-        @file.write "%016x\t%08x" % [pos, ino]
+        @file.write "%016x\t%016x" % [pos, ino]
       end
 
       def update_pos(pos)
@@ -663,7 +666,7 @@ module Fluent
 
       def read_inode
         @file.pos = @seek + INO_OFFSET
-        raw = @file.read(8)
+        raw = @file.read(16)
         raw ? raw.to_i(16) : 0
       end
 

data/lib/fluent/supervisor.rb

@@ -323,10 +323,16 @@ module Fluent
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         $log.info "restarting"
-        if pid = @main_pid
-          Process.kill(:TERM, pid)
-          # don't resuce Erro::ESRSH here (invalid status)
-        end
+        # Creating new thread due to mutex can't lock
+        # in main thread during trap context
+        Thread.new {
+          read_config
+          apply_system_config
+          if pid = @main_pid
+            Process.kill(:TERM, pid)
+            # don't resuce Erro::ESRSH here (invalid status)
+          end
+        }.run
       end
 
       trap :USR1 do

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.58'
+  VERSION = '0.10.59'
 
 end

data/test/test_output.rb

@@ -1,9 +1,12 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/output'
+require 'timecop'
+require 'flexmock'
 
 module FluentOutputTest
   include Fluent
+  include FlexMock::TestCase
 
   class BufferedOutputTest < ::Test::Unit::TestCase
     include FluentOutputTest
@@ -42,6 +45,10 @@ module FluentOutputTest
       # disable_retry_limit
       d = create_driver(CONFIG + %[disable_retry_limit true])
       assert_equal true, d.instance.disable_retry_limit
+
+      # retry_wait is converted to Float for calc_retry_wait
+      d = create_driver(CONFIG + %[retry_wait 1s])
+      assert_equal Float, d.instance.retry_wait.class
     end
 
     def test_calc_retry_wait
@@ -62,6 +69,14 @@ module FluentOutputTest
       assert_equal 4, d.instance.calc_retry_wait
     end
 
+    def test_calc_retry_wait_with_integer_retry_wait
+      d = create_driver(CONFIG + %[retry_wait 2s])
+      d.instance.retry_limit.times {
+        d.instance.instance_eval { @num_errors += 1 }
+      }
+      assert_equal true, d.instance.calc_retry_wait.finite?
+    end
+
     def test_large_num_retries
       # Test that everything works properly after a very large number of
       # retries and we hit the expected max_retry_wait.
@@ -146,5 +161,77 @@ module FluentOutputTest
       assert_true d.instance.respond_to?(:router=)
       assert_true d.instance.router.respond_to?(:emit)
     end
+
+    sub_test_case "test_force_flush" do
+      setup do
+        time = Time.parse("2011-01-02 13:14:15 UTC")
+        Timecop.freeze(time)
+        @time = time.to_i
+      end
+
+      teardown do
+        Timecop.return
+      end
+
+      test "force_flush works on retrying" do
+        d = create_driver(CONFIG)
+        d.instance.start
+        buffer = d.instance.instance_variable_get(:@buffer)
+        # imitate 10 failures
+        d.instance.instance_variable_set(:@num_errors, 10)
+        d.instance.instance_variable_set(:@next_retry_time, @time + d.instance.calc_retry_wait)
+        # buffer should be popped (flushed) immediately
+        flexmock(buffer).should_receive(:pop).once
+        # force_flush
+        buffer.emit("test", 'test', NullOutputChain.instance)
+        d.instance.force_flush
+        10.times { sleep 0.05 }
+      end
+    end
+  end
+
+  class TimeSlicedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
+    include FlexMock::TestCase
+
+    def setup
+      Fluent::Test.setup
+      FileUtils.rm_rf(TMP_DIR)
+      FileUtils.mkdir_p(TMP_DIR)
+    end
+
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/time_sliced_output")
+
+    CONFIG = %[]
+
+    def create_driver(conf=CONFIG)
+      Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::TimeSlicedOutput).configure(conf, true)
+    end
+
+    sub_test_case "test_force_flush" do
+      setup do
+        time = Time.parse("2011-01-02 13:14:15 UTC")
+        Timecop.freeze(time)
+        @es = OneEventStream.new(time.to_i, {"message" => "foo"})
+      end
+
+      teardown do
+        Timecop.return
+      end
+
+      test "force_flush immediately flushes" do
+        d = create_driver(CONFIG + %[
+          time_format %Y%m%d%H%M%S
+          buffer_path #{TMP_DIR}/foo
+        ])
+        d.instance.start
+        # buffer should be popped (flushed) immediately
+        flexmock(d.instance.instance_variable_get(:@buffer)).should_receive(:pop).once
+        # force_flush
+        d.instance.emit('test', @es, NullOutputChain.instance)
+        d.instance.force_flush
+        10.times { sleep 0.05 }
+      end
+    end
   end
 end

data/test/test_parser.rb

@@ -199,25 +199,33 @@ module ParserTest
 
     def setup
       @parser = TextParser::ApacheParser.new
+      @expected = {
+        'user'    => nil,
+        'method'  => 'GET',
+        'code'    => 200,
+        'size'    => 777,
+        'host'    => '192.168.0.1',
+        'path'    => '/',
+        'referer' => nil,
+        'agent'   => 'Opera/12.0'
+      }
     end
 
     def test_parse
       @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
         assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
-        assert_equal({
-          'user'    => nil,
-          'method'  => 'GET',
-          'code'    => 200,
-          'size'    => 777,
-          'host'    => '192.168.0.1',
-          'path'    => '/',
-          'referer' => nil,
-          'agent'   => 'Opera/12.0'
-        }, record)
+        assert_equal(@expected, record)
       }
       assert_equal(TextParser::ApacheParser::REGEXP, @parser.patterns['format'])
       assert_equal(TextParser::ApacheParser::TIME_FORMAT, @parser.patterns['time_format'])
     end
+
+    def test_parse_without_http_version
+      @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(@expected, record)
+      }
+    end
   end
 
   class SyslogParserTest < ::Test::Unit::TestCase
@@ -354,6 +362,13 @@ module ParserTest
         assert_equal(@expected.merge('path' => '/a[ ]b'), record)
       }
     end
+
+    def test_parse_without_http_version
+      @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(@expected, record)
+      }
+    end
   end
 
   class TSVParserTest < ::Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.58
+  version: 0.10.59
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-14 00:00:00.000000000 Z
+date: 2015-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack