fluentd 0.12.2 → 0.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7788505e84495465d06f359b75401362f04caaa
-  data.tar.gz: 47a5ed1ef3ee3b55cabbcfdf65877374e66642b7
+  metadata.gz: 5a5677f5a62821f6d9300f21eddded90ff1f0e0a
+  data.tar.gz: 004b3bb49cd860b6132811b3cf1aa7ef89c4611f
 SHA512:
-  metadata.gz: 7b174bf87334d6c06db915f3c244e39dbc637c7f21872fc227046f9b8c4e332809972e956013c0f0c4fea1ed187e89a45656723d698e733f8fcf78e85cf8f7d0
-  data.tar.gz: 602154651def0f60ddc693b6ba1b1bd5c7d7ce0f2efbc7b4413ba16905929239fb7928e41fd00f93838f4a3ee909f5978737dded60b2fcf6c8fd035221cc381d
+  metadata.gz: 2aa4d2ef65d85eeb6df07245c6f9459b17d1a8d1546862ad7b96556d413a029dc3fff2514f49ffd1f3f307f11cf117816fcfcf28060f823525a01392c4e342fc
+  data.tar.gz: ff625fbf8a5b70866d413b6c5e5f1b8d4ebb52acf696ee1416de2cd249ff6ca14c223a312d633027f6c21e2e7a95b150abc3a43baa116ab54b55856543375432

data/.gitignore

@@ -19,3 +19,4 @@ pkg/*
 test/tmp/*
 test/config/tmp/*
 make_dist.sh
+Gemfile.local

data/.travis.yml

@@ -4,6 +4,7 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1
+  - 2.2
   - ruby-head
   - rbx-2
 
@@ -25,3 +26,4 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx-2
+    - rvm: 2.2

data/ChangeLog

@@ -1,6 +1,22 @@
 # v0.12
 
-## Release 0.12.1 - 2014/12/14
+## Release 0.12.3 - 2015/01/16
+
+### New features / Enhancement
+
+* parser: nginx and apache2 format can now parse access log without http-version
+* filter_record_transformer: Allow non-string values like array / hash
+* plugin: Add writable check for path related parameter
+* command: -g and -G now works in embedded Ruby environment like Treasure Agent
+
+### Bug fixes
+
+* buffer: Fix to flush a buffer by USR1 signal even on retrying
+* buffer: Fix TimeSlicedOutput doesn't flush with SIGUSR1
+* buffer: Fix broken calc_retry_wait if Integer is used for retry_wait parameter
+* engine: Fix SIGHUP does not reload config
+
+## Release 0.12.2 - 2014/12/20
 
 ### New Features
 

data/Gemfile

@@ -1,3 +1,9 @@
 source 'https://rubygems.org/'
 
 gemspec
+
+local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
+if File.exist?(local_gemfile)
+  puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v`
+  instance_eval File.read(local_gemfile)
+end

data/lib/fluent/command/bundler_injection.rb

@@ -14,18 +14,30 @@
 #    limitations under the License.
 #
 
-system("bundle install")
-unless $?.success?
-  exit $?.exitstatus
-end
-
-cmdline = [
-  'bundle',
-  'exec',
-  RbConfig.ruby,
-  File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
-] + ARGV
+if ENV['BUNDLE_BIN_PATH']
+  puts 'error: You seem to use `bundle exec` already.'
+  exit 1
+else
+  begin
+    bundle_bin = Gem::Specification.find_by_name('bundler').bin_file('bundle')
+  rescue Gem::LoadError => e
+    puts "error: #{e}"
+    exit 1
+  end
+  ruby_bin = RbConfig.ruby
+  system("#{ruby_bin} #{bundle_bin} install")
+  unless $?.success?
+    exit $?.exitstatus
+  end
 
-exec *cmdline
-exit! 127
+  cmdline = [
+    ruby_bin,
+    bundle_bin,
+    'exec',
+    ruby_bin,
+    File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
+  ] + ARGV
 
+  exec *cmdline
+  exit! 127
+end

data/lib/fluent/output.rb

@@ -192,6 +192,7 @@ module Fluent
     def configure(conf)
       super
 
+      @retry_wait = @retry_wait.to_f # converted to Float for calc_retry_wait
       @buffer = Plugin.new_buffer(@buffer_type)
       @buffer.configure(conf)
 
@@ -272,7 +273,7 @@ module Fluent
     #def write(chunk)
     #end
 
-    def enqueue_buffer
+    def enqueue_buffer(force = false)
       @buffer.keys.each {|key|
         @buffer.push(key)
       }
@@ -383,7 +384,10 @@ module Fluent
     end
 
     def force_flush
-      enqueue_buffer
+      @num_errors_lock.synchronize do
+        @next_retry_time = Engine.now - 1
+      end
+      enqueue_buffer(true)
       submit_flush
     end
 
@@ -551,8 +555,14 @@ module Fluent
       }
     end
 
-    def enqueue_buffer
-      @enqueue_buffer_proc.call
+    def enqueue_buffer(force = false)
+      if force
+        @buffer.keys.each {|key|
+          @buffer.push(key)
+        }
+      else
+        @enqueue_buffer_proc.call
+      end
     end
 
     #def format(tag, event)

data/lib/fluent/parser.rb

@@ -407,7 +407,7 @@ module Fluent
     end
 
     class ApacheParser < Parser
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
@@ -631,7 +631,7 @@ module Fluent
       'tsv' => Proc.new { TSVParser.new },
       'ltsv' => Proc.new { LabeledTSVParser.new },
       'csv' => Proc.new { CSVParser.new },
-      'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+      'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
       'none' => Proc.new { NoneParser.new },
       'multiline' => Proc.new { MultilineParser.new },
     }.each { |name, factory|

data/lib/fluent/plugin/buf_file.rb

@@ -83,6 +83,7 @@ module Fluent
     end
 
     config_param :buffer_path, :string
+    config_param :flush_at_shutdown, :bool, :default => false
 
     # 'symlink_path' is currently only for out_file.
     # That is the reason why this is not config_param, but attr_accessor.
@@ -106,11 +107,6 @@ module Fluent
         @buffer_path_suffix = ".log"
       end
 
-      if flush_at_shutdown = conf['flush_at_shutdown']
-        @flush_at_shutdown = true
-      else
-        @flush_at_shutdown = false
-      end
     end
 
     def start

data/lib/fluent/plugin/file_util.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module FileUtil
+    # Check file is writable if file exists
+    # Check directory is writable if file does not exist
+    #
+    # @param [String] path File path
+    # @return [Boolean] file is writable or not
+    def writable?(path)
+      path = File.exist?(path) ? path : File.dirname(path)
+      File.writable?(path)
+    end
+    module_function :writable?
+  end
+end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -38,7 +38,7 @@ module Fluent
       conf.elements.select { |element| element.name == 'record' }.each do |element|
         element.each_pair do |k, v|
           element.has_key?(k) # to suppress unread configuration warning
-          @map[k] = v
+          @map[k] = parse_value(v)
         end
       end
 
@@ -92,17 +92,45 @@ module Fluent
 
     private
 
+    def parse_value(value_str)
+      if value_str.start_with?('{', '[')
+        JSON.parse(value_str)
+      else
+        value_str
+      end
+    rescue => e
+      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", :error_class => e.class, :error => e.message
+      value_str # emit as string
+    end
+
     def reform(time, record, opts)
       @placeholder_expander.prepare_placeholders(time, record, opts)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
-      @map.each_pair {|k, v| new_record[k] = @placeholder_expander.expand(v) }
+      @map.each_pair {|k, v| new_record[k] = interpolate(v)}
       @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end
 
+    def interpolate(value)
+      if value.is_a?(String)
+        value = @placeholder_expander.expand(value)
+      elsif value.is_a?(Hash)
+        new_value = {}
+        value.each_pair do |k, v|
+          new_value[@placeholder_expander.expand(k)] = interpolate(v)
+        end
+        value = new_value
+      elsif value.is_a?(Array)
+        value.each_with_index do |v, i|
+          value[i] = interpolate(v)
+        end
+      end
+      value
+    end
+
     def tag_prefix(tag_parts)
       return [] if tag_parts.empty?
       tag_prefix = [tag_parts.first]
@@ -175,18 +203,13 @@ module Fluent
         @placeholders = struct
       end
 
-      # Replace placeholders in a string
-      #
-      # @param [String] str         the string to be replaced
       def expand(str)
-        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-        begin
-          eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
-        rescue => e
-          log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
-          log.warn_backtrace
-          nil
-        end
+        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..} 
+        eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
+      rescue => e
+        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
+        log.warn_backtrace
+        nil
       end
 
       class UndefOpenStruct < OpenStruct

data/lib/fluent/plugin/in_debug_agent.rb

@@ -20,6 +20,7 @@ module Fluent
 
     def initialize
       require 'drb/drb'
+      require 'fluent/plugin/file_util'
       super
     end
 
@@ -31,6 +32,11 @@ module Fluent
 
     def configure(conf)
       super
+      if @unix_path
+        unless ::Fluent::FileUtil.writable?(@unix_path)
+          raise ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
+        end
+      end
     end
 
     def start

data/lib/fluent/plugin/out_file.rb

@@ -38,6 +38,7 @@ module Fluent
     def initialize
       require 'zlib'
       require 'time'
+      require 'fluent/plugin/file_util'
       super
     end
 
@@ -59,6 +60,11 @@ module Fluent
         conf['buffer_path'] ||= "#{@path}.*"
       end
 
+      test_path = generate_path(Time.now.strftime(@time_slice_format))
+      unless ::Fluent::FileUtil.writable?(test_path)
+        raise ConfigError, "out_file: `#{test_path}` is not writable"
+      end
+
       super
 
       @formatter = Plugin.new_formatter(@format)
@@ -72,7 +78,7 @@ module Fluent
     end
 
     def write(chunk)
-      path = generate_path(chunk)
+      path = generate_path(chunk.key)
       FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
 
       case @compress
@@ -97,21 +103,23 @@ module Fluent
 
     private
 
-    def generate_path(chunk)
+    def suffix
       case @compress
       when nil
-        suffix = ''
+        ''
       when :gz
-        suffix = ".gz"
+        ".gz"
       end
+    end
 
+    def generate_path(time_string)
       if @append
-        "#{@path_prefix}#{chunk.key}#{@path_suffix}#{suffix}"
+        "#{@path_prefix}#{time_string}#{@path_suffix}#{suffix}"
       else
         path = nil
         i = 0
         begin
-          path = "#{@path_prefix}#{chunk.key}_#{i}#{@path_suffix}#{suffix}"
+          path = "#{@path_prefix}#{time_string}_#{i}#{@path_suffix}#{suffix}"
           i += 1
         end while File.exist?(path)
         path

data/lib/fluent/supervisor.rb

@@ -321,6 +321,8 @@ module Fluent
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         $log.info "restarting"
+        read_config
+        apply_system_config
         if pid = @main_pid
           Process.kill(:TERM, pid)
           # don't resuce Erro::ESRSH here (invalid status)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.2'
+  VERSION = '0.12.3'
 
 end

data/test/plugin/test_file_util.rb

@@ -0,0 +1,39 @@
+require_relative '../helper'
+require 'fluent/plugin/file_util'
+require 'fileutils'
+
+class FileUtilTest < Test::Unit::TestCase
+  def setup
+    FileUtils.rm_rf(TEST_DIR)
+    FileUtils.mkdir_p(TEST_DIR)
+  end
+
+  TEST_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/file_util")
+
+  sub_test_case 'writable?' do
+    test 'file exists and writable' do
+      FileUtils.touch("#{TEST_DIR}/test_file")
+      assert_true Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    end
+
+    test 'file exists and not writable' do
+      FileUtils.touch("#{TEST_DIR}/test_file")
+      File.chmod(0444, "#{TEST_DIR}/test_file")
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    end
+
+    test 'file does not exist and directory is writable' do
+      assert_true Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    end
+
+    test 'file does not exist and directory is not writable' do
+      File.chmod(0444, TEST_DIR)
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    end
+
+    test 'directory does not exist' do
+      FileUtils.rm_rf(TEST_DIR)
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    end
+  end
+end

data/test/plugin/test_filter_record_transformer.rb

@@ -115,6 +115,45 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
       end
     end
+
+    test 'hash_value' do
+      config = %[
+        <record>
+          hash_field {"k1":100, "k2":"foobar"}
+        </record>
+      %]
+      msgs = ['1', '2']
+      es = emit(config, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_equal({"k1"=>100, "k2"=>"foobar"}, r['hash_field'])
+      end
+    end
+
+    test 'array_value' do
+      config = %[
+        <record>
+          array_field [1, 2, 3]
+        </record>
+      %]
+      msgs = ['1', '2']
+      es = emit(config, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_equal([1,2,3], r['array_field'])
+      end
+    end
+
+    test 'array_hash_mixed' do
+      config = %[
+        <record>
+          mixed_field {"hello":[1,2,3], "world":{"foo":"bar"}}
+        </record>
+      %]
+      msgs = ['1', '2']
+      es = emit(config, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_equal({"hello"=>[1,2,3], "world"=>{"foo"=>"bar"}}, r['mixed_field'])
+      end
+    end
   end
 
   sub_test_case 'test placeholders' do
@@ -214,6 +253,50 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           assert_equal("bar #{msgs[i]}", r['message'])
         end
       end
+
+      test "hash values with placeholders with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            hash_field {
+              "hostname":"${hostname}",
+              "tag":"${tag}",
+              "${tag}":100
+            }
+          </record>
+        ]
+        msgs = ['1', '2']
+        es = emit(config, msgs)
+        es.each_with_index do |(t, r), i|
+          assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, r['hash_field'])
+        end
+      end
+      test "array values with placeholders with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            array_field ["${hostname}", "${tag}"]
+          </record>
+        ]
+        msgs = ['1', '2']
+        es = emit(config, msgs)
+        es.each_with_index do |(t, r), i|
+          assert_equal([@hostname, @tag], r['array_field'])
+        end
+      end
+      test "array and hash values with placeholders with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            mixed_field [{"tag":"${tag}"}]
+          </record>
+        ]
+        msgs = ['1', '2']
+        es = emit(config, msgs)
+        es.each_with_index do |(t, r), i|
+          assert_equal([{"tag" => @tag}], r['mixed_field'])
+        end
+      end
     end
 
     test 'unknown placeholder (enable_ruby no)' do

data/test/plugin/test_in_debug_agent.rb

@@ -0,0 +1,26 @@
+require_relative '../helper'
+require 'fileutils'
+
+class DebugAgentInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/in_debug_agent")
+
+  def create_driver(conf = '')
+    Fluent::Test::InputTestDriver.new(Fluent::DebugAgentInput).configure(conf)
+  end
+
+  def test_unix_path_writable
+    assert_nothing_raised do
+      create_driver %[unix_path #{TMP_DIR}/test_path]
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[unix_path #{TMP_DIR}/does_not_exist/test_path]
+    end
+  end
+end

data/test/plugin/test_out_file.rb

@@ -40,6 +40,16 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal :gz, d.instance.compress
   end
 
+  def test_path_writable
+    assert_nothing_raised do
+      create_driver %[path #{TMP_DIR}/test_path]
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[path #{TMP_DIR}/does_not_exist/test_path]
+    end
+  end
+
   def test_default_localtime
     d = create_driver(%[path #{TMP_DIR}/out_file_test])
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i

data/test/test_output.rb

@@ -1,9 +1,12 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/output'
+require 'timecop'
+require 'flexmock'
 
 module FluentOutputTest
   include Fluent
+  include FlexMock::TestCase
 
   class BufferedOutputTest < ::Test::Unit::TestCase
     include FluentOutputTest
@@ -53,6 +56,10 @@ module FluentOutputTest
       # disable_retry_limit
       d = create_driver(CONFIG + %[disable_retry_limit true])
       assert_equal true, d.instance.disable_retry_limit
+
+      # retry_wait is converted to Float for calc_retry_wait
+      d = create_driver(CONFIG + %[retry_wait 1s])
+      assert_equal Float, d.instance.retry_wait.class
     end
 
     def test_calc_retry_wait
@@ -73,6 +80,14 @@ module FluentOutputTest
       assert_equal 4, d.instance.calc_retry_wait
     end
 
+    def test_calc_retry_wait_with_integer_retry_wait
+      d = create_driver(CONFIG + %[retry_wait 2s])
+      d.instance.retry_limit.times {
+        d.instance.instance_eval { @num_errors += 1 }
+      }
+      assert_equal true, d.instance.calc_retry_wait.finite?
+    end
+
     def test_large_num_retries
       # Test that everything works properly after a very large number of
       # retries and we hit the expected max_retry_wait.
@@ -159,5 +174,77 @@ module FluentOutputTest
       ])
       assert_not_nil d.instance.instance_variable_get(:@secondary).router
     end
+
+    sub_test_case "test_force_flush" do
+      setup do
+        time = Time.parse("2011-01-02 13:14:15 UTC")
+        Timecop.freeze(time)
+        @time = time.to_i
+      end
+
+      teardown do
+        Timecop.return
+      end
+
+      test "force_flush works on retrying" do
+        d = create_driver(CONFIG)
+        d.instance.start
+        buffer = d.instance.instance_variable_get(:@buffer)
+        # imitate 10 failures
+        d.instance.instance_variable_set(:@num_errors, 10)
+        d.instance.instance_variable_set(:@next_retry_time, @time + d.instance.calc_retry_wait)
+        # buffer should be popped (flushed) immediately
+        flexmock(buffer).should_receive(:pop).once
+        # force_flush
+        buffer.emit("test", 'test', NullOutputChain.instance)
+        d.instance.force_flush
+        10.times { sleep 0.05 }
+      end
+    end
+  end
+
+  class TimeSlicedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
+    include FlexMock::TestCase
+
+    def setup
+      Fluent::Test.setup
+      FileUtils.rm_rf(TMP_DIR)
+      FileUtils.mkdir_p(TMP_DIR)
+    end
+
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/time_sliced_output")
+
+    CONFIG = %[]
+
+    def create_driver(conf=CONFIG)
+      Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::TimeSlicedOutput).configure(conf, true)
+    end
+
+    sub_test_case "test_force_flush" do
+      setup do
+        time = Time.parse("2011-01-02 13:14:15 UTC")
+        Timecop.freeze(time)
+        @es = OneEventStream.new(time.to_i, {"message" => "foo"})
+      end
+
+      teardown do
+        Timecop.return
+      end
+
+      test "force_flush immediately flushes" do
+        d = create_driver(CONFIG + %[
+          time_format %Y%m%d%H%M%S
+          buffer_path #{TMP_DIR}/foo
+        ])
+        d.instance.start
+        # buffer should be popped (flushed) immediately
+        flexmock(d.instance.instance_variable_get(:@buffer)).should_receive(:pop).once
+        # force_flush
+        d.instance.emit('test', @es, NullOutputChain.instance)
+        d.instance.force_flush
+        10.times { sleep 0.05 }
+      end
+    end
   end
 end

data/test/test_parser.rb

@@ -199,25 +199,33 @@ module ParserTest
 
     def setup
       @parser = TextParser::ApacheParser.new
+      @expected = {
+        'user'    => nil,
+        'method'  => 'GET',
+        'code'    => 200,
+        'size'    => 777,
+        'host'    => '192.168.0.1',
+        'path'    => '/',
+        'referer' => nil,
+        'agent'   => 'Opera/12.0'
+      }
     end
 
     def test_parse
       @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
         assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
-        assert_equal({
-          'user'    => nil,
-          'method'  => 'GET',
-          'code'    => 200,
-          'size'    => 777,
-          'host'    => '192.168.0.1',
-          'path'    => '/',
-          'referer' => nil,
-          'agent'   => 'Opera/12.0'
-        }, record)
+        assert_equal(@expected, record)
       }
       assert_equal(TextParser::ApacheParser::REGEXP, @parser.patterns['format'])
       assert_equal(TextParser::ApacheParser::TIME_FORMAT, @parser.patterns['time_format'])
     end
+
+    def test_parse_without_http_version
+      @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(@expected, record)
+      }
+    end
   end
 
   class SyslogParserTest < ::Test::Unit::TestCase
@@ -354,6 +362,13 @@ module ParserTest
         assert_equal(@expected.merge('path' => '/a[ ]b'), record)
       }
     end
+
+    def test_parse_without_http_version
+      @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(@expected, record)
+      }
+    end
   end
 
   class TSVParserTest < ::Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.2
+  version: 0.12.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-19 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -339,6 +339,7 @@ files:
 - lib/fluent/plugin/buf_file.rb
 - lib/fluent/plugin/buf_memory.rb
 - lib/fluent/plugin/exec_util.rb
+- lib/fluent/plugin/file_util.rb
 - lib/fluent/plugin/filter_grep.rb
 - lib/fluent/plugin/filter_record_transformer.rb
 - lib/fluent/plugin/in_debug_agent.rb
@@ -396,8 +397,10 @@ files:
 - test/plugin/data/log/test.log
 - test/plugin/test_buf_file.rb
 - test/plugin/test_buf_memory.rb
+- test/plugin/test_file_util.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_record_transformer.rb
+- test/plugin/test_in_debug_agent.rb
 - test/plugin/test_in_dummy.rb
 - test/plugin/test_in_exec.rb
 - test/plugin/test_in_forward.rb
@@ -477,8 +480,10 @@ test_files:
 - test/plugin/data/log/test.log
 - test/plugin/test_buf_file.rb
 - test/plugin/test_buf_memory.rb
+- test/plugin/test_file_util.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_record_transformer.rb
+- test/plugin/test_in_debug_agent.rb
 - test/plugin/test_in_dummy.rb
 - test/plugin/test_in_exec.rb
 - test/plugin/test_in_forward.rb