fluentd 1.9.2-x64-mingw32 → 1.9.3-x64-mingw32

data/lib/fluent/plugin/in_unix.rb

@@ -28,12 +28,6 @@ module Fluent
   class StreamInput < Input
     config_param :blocking_timeout, :time, default: 0.5
 
-    def initialize
-      require 'socket'
-      require 'yajl'
-      super
-    end
-
     def start
       super
 

data/lib/fluent/plugin/out_file.rb

@@ -17,7 +17,6 @@
 require 'fileutils'
 require 'zlib'
 require 'time'
-require 'tempfile'
 
 require 'fluent/plugin/output'
 require 'fluent/config/error'
@@ -37,9 +36,6 @@ module Fluent::Plugin
       gzip: :gzip,
     }
 
-    FILE_PERMISSION = 0644
-    DIR_PERMISSION = 0755
-
     DEFAULT_TIMEKEY = 60 * 60 * 24
 
     desc "The Path of the file."
@@ -182,8 +178,8 @@ module Fluent::Plugin
         end
       end
 
-      @dir_perm = system_config.dir_permission || DIR_PERMISSION
-      @file_perm = system_config.file_permission || FILE_PERMISSION
+      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
+      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
     end
 
@@ -232,28 +228,10 @@ module Fluent::Plugin
     end
 
     def write_gzip_with_compression(path, chunk)
-      if @append
-        # This code will be removed after zlib/multithread bug is fixed.
-        # Use Tempfile to avoid broken gzip files: https://github.com/fluent/fluentd/issues/1903
-        Tempfile.create('out_file-gzip-append') { |temp|
-          begin
-            writer = Zlib::GzipWriter.new(temp)
-            chunk.write_to(writer, compressed: :text)
-          ensure
-            writer.finish # avoid zlib finalizer warning
-          end
-          temp.rewind
-
-          File.open(path, "ab", @file_perm) do |f|
-            IO.copy_stream(temp, f)
-          end
-        }
-      else
-        File.open(path, "ab", @file_perm) do |f|
-          gz = Zlib::GzipWriter.new(f)
-          chunk.write_to(gz, compressed: :text)
-          gz.close
-        end
+      File.open(path, "ab", @file_perm) do |f|
+        gz = Zlib::GzipWriter.new(f)
+        chunk.write_to(gz, compressed: :text)
+        gz.close
       end
     end
 

data/lib/fluent/plugin/out_secondary_file.rb

@@ -23,8 +23,6 @@ module Fluent::Plugin
   class SecondaryFileOutput < Output
     Fluent::Plugin.register_output("secondary_file", self)
 
-    FILE_PERMISSION = 0644
-    DIR_PERMISSION = 0755
     PLACEHOLDER_REGEX = /\${(tag(\[\d+\])?|[\w.@-]+)}/
 
     desc "The directory path of the output file."
@@ -61,8 +59,8 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
       end
 
-      @dir_perm = system_config.dir_permission || DIR_PERMISSION
-      @file_perm = system_config.file_permission || FILE_PERMISSION
+      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
+      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
     end
 
     def multi_workers_ready?

data/lib/fluent/plugin/output.rb

@@ -1203,7 +1203,7 @@ module Fluent
           backup_dir = File.dirname(backup_file)
 
           log.warn "bad chunk is moved to #{backup_file}"
-          FileUtils.mkdir_p(backup_dir) unless Dir.exist?(backup_dir)
+          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(backup_dir)
           File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
             chunk.write_to(f)
           }

data/lib/fluent/plugin/parser_multiline.rb

@@ -24,9 +24,33 @@ module Fluent
 
       desc 'Specify regexp pattern for start line of multiple lines'
       config_param :format_firstline, :string, default: nil
+      desc 'Enable an option returning line as unmatched_line'
+      config_param :unmatched_lines, :string, default: nil
 
       FORMAT_MAX_NUM = 20
 
+      class MultilineRegexpParser < Fluent::Plugin::RegexpParser
+        def parse(text)
+          m = @expression.match(text)
+          unless m
+            yield nil, nil
+            return m
+          end
+
+          r = {}
+          m.names.each do |name|
+            if (value = m[name])
+              r[name] = value
+            end
+          end
+
+          time, record = convert_values(parse_time(r), r)
+
+          yield(time, record)
+          m
+        end
+      end
+
       def configure(conf)
         super
 
@@ -37,7 +61,7 @@ module Fluent
             raise "No named captures"
           end
           regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/m" }, [])
-          @parser = Fluent::Plugin::RegexpParser.new
+          @parser = Fluent::Plugin::MultilineParser::MultilineRegexpParser.new
           @parser.configure(conf + regexp_conf)
         rescue => e
           raise Fluent::ConfigError, "Invalid regexp '#{formats}': #{e}"
@@ -50,7 +74,29 @@ module Fluent
       end
 
       def parse(text, &block)
-        @parser.call(text, &block)
+        loop do
+          m =
+            if @unmatched_lines
+              @parser.call(text) do |time, record|
+                if time && record
+                  yield(time, record)
+                else
+                  yield(Fluent::EventTime.now, { 'unmatched_line' => text })
+                end
+              end
+            else
+              @parser.call(text, &block)
+            end
+
+          return if m.nil?
+
+          text = m.post_match
+          if text.start_with?("\n")
+            text = text[1..-1]
+          end
+
+          return if text.empty?
+        end
       end
 
       def has_firstline?

data/lib/fluent/plugin_helper/server.rb

@@ -706,9 +706,13 @@ module Fluent
 
                 return true
               end
-            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
+            rescue OpenSSL::SSL::SSLError => e
+              # Use same log level as on_readable
+              @log.warn "unexpected error before accepting TLS connection by OpenSSL", error: e
+              close rescue nil
             end
 
             false

data/lib/fluent/plugin_id.rb

@@ -76,7 +76,7 @@ module Fluent
 
       # Fluent::Plugin::Base#fluentd_worker_id
       dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
-      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
       @_plugin_root_dir = dir.freeze
       dir
     end

data/lib/fluent/supervisor.rb

@@ -439,9 +439,13 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
+
+        if @path && log_dir_perm
+          File.chmod(log_dir_perm || 0755, File.dirname(@path))
+        end
       end
 
       def level=(level)
@@ -529,7 +533,7 @@ module Fluent
           end
         else
           begin
-            FileUtils.mkdir_p(root_dir)
+            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
           rescue => e
             raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end
@@ -620,7 +624,11 @@ module Fluent
       @system_config = build_system_config(@conf)
 
       @log.level = @system_config.log_level
-      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+      @log.apply_options(
+        format: @system_config.log.format,
+        time_format: @system_config.log.time_format,
+        log_dir_perm: @system_config.dir_permission,
+      )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.9.2'
+  VERSION = '1.9.3'
 
 end

data/test/command/test_fluentd.rb

@@ -46,8 +46,13 @@ class TestFluentdCommand < ::Test::Unit::TestCase
   end
 
   def create_cmdline(conf_path, *fluentd_options)
-    cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
-    ["bundle", "exec", cmd_path, "-c", conf_path, *fluentd_options]
+    if Fluent.windows?
+      cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
+      ["bundle", "exec", ServerEngine.ruby_bin_path, cmd_path, "-c", conf_path, *fluentd_options]
+    else
+      cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
+      ["bundle", "exec", cmd_path, "-c", conf_path, *fluentd_options]
+    end
   end
 
   def execute_command(cmdline, chdir=TMP_DIR, env = {})
@@ -694,6 +699,28 @@ CONF
       )
     end
 
+    test 'success to start a worker2 with worker specific configuration' do
+      conf = <<CONF
+<system>
+  root_dir #{@root_path}
+  dir_permission 0744
+</system>
+CONF
+      conf_path = create_conf_file('worker_section0.conf', conf)
+
+      FileUtils.rm_rf(@root_path) rescue nil
+
+      assert_path_not_exist(@root_path)
+      assert_log_matches(create_cmdline(conf_path), 'spawn command to main') # any message is ok
+      assert_path_exist(@root_path)
+      if Fluent.windows?
+        # In Windows, dir permission is always 755.
+        assert_equal '755', File.stat(@root_path).mode.to_s(8)[-3, 3]
+      else
+        assert_equal '744', File.stat(@root_path).mode.to_s(8)[-3, 3]
+      end
+    end
+
     test 'success to start a worker with worker specific configuration' do
       conf = <<CONF
 <system>
@@ -804,6 +831,7 @@ CONF
       '-internal-encoding' => '--internal-encoding=utf-8',
     )
     test "-E option is set to RUBYOPT3" do |opt|
+      omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
       conf = <<CONF
 <source>
   @type dummy
@@ -837,6 +865,7 @@ CONF
     end
 
     test 'invalid values are set to RUBYOPT' do
+      omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
       conf = <<CONF
 <source>
   @type dummy

data/test/plugin/test_buf_file.rb

@@ -23,8 +23,10 @@ module FluentPluginFileBufferTest
 end
 
 class FileBufferTest < Test::Unit::TestCase
-  def metadata(timekey: nil, tag: nil, variables: nil)
-    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  def metadata(timekey: nil, tag: nil, variables: nil, seq: 0)
+    m = Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+    m.seq = seq
+    m
   end
 
   def write_metadata_old(path, chunk_id, metadata, size, ctime, mtime)
@@ -43,6 +45,7 @@ class FileBufferTest < Test::Unit::TestCase
   def write_metadata(path, chunk_id, metadata, size, ctime, mtime)
     metadata = {
       timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables,
+      seq: metadata.seq,
       id: chunk_id,
       s: size,
       c: ctime,
@@ -278,7 +281,7 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal m1, c1.metadata
       assert c1.empty?
       assert_equal :unstaged, c1.state
-      assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c1.permission
+      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
       assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c1.unique_id)}."), c1.path
       assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
 
@@ -288,7 +291,7 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal m2, c2.metadata
       assert c2.empty?
       assert_equal :unstaged, c2.state
-      assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c2.permission
+      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c2.permission
       assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path
       assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') }
 
@@ -996,6 +999,83 @@ class FileBufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'there are the same timekey metadata in stage' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+      @bufpath = File.join(@bufdir, 'testbuf.*.log')
+      FileUtils.rm_r(@bufdir) if File.exist?(@bufdir)
+      FileUtils.mkdir_p(@bufdir)
+
+      m = metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i)
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c1id)}.log")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay1"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay2"}].to_json + "\n"
+      end
+      write_metadata(p1 + '.meta', c1id, m, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)
+
+      c2id = Fluent::UniqueId.generate
+      p2 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c2id)}.log")
+      File.open(p2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay3"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay4"}].to_json + "\n"
+      end
+      m2 = m.dup_next
+      write_metadata(p2 + '.meta', c2id, m2, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)
+
+      c3id = Fluent::UniqueId.generate
+      p3 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c3id)}.log")
+      File.open(p3, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay5"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay6"}].to_json + "\n"
+      end
+      m3 = m2.dup_next
+      write_metadata(p3 + '.meta', c3id, m3, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)
+
+      c4id = Fluent::UniqueId.generate
+      p4 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c4id)}.log")
+      File.open(p4, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay5"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay6"}].to_json + "\n"
+      end
+      write_metadata(p4 + '.meta', c4id, m3, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test '#resume returns each chunks' do
+      s, e = @p.resume
+      assert_equal 3, s.size
+      assert_equal [0, 1, 2], s.keys.map(&:seq).sort
+      assert_equal 1, e.size
+      assert_equal [0], e.map { |e| e.metadata.seq }
+    end
+  end
+
   sub_test_case 'there are some non-buffer chunk files, with a path without buffer chunk ids' do
     setup do
       @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)

data/test/plugin/test_buf_file_single.rb

@@ -239,7 +239,7 @@ class FileSingleBufferTest < Test::Unit::TestCase
       assert_equal m1, c1.metadata
       assert c1.empty?
       assert_equal :unstaged, c1.state
-      assert_equal Fluent::Plugin::Buffer::FileSingleChunk::FILE_PERMISSION, c1.permission
+      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
       assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path
       assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
 
@@ -258,7 +258,7 @@ class FileSingleBufferTest < Test::Unit::TestCase
       assert_equal m1, c1.metadata
       assert c1.empty?
       assert_equal :unstaged, c1.state
-      assert_equal Fluent::Plugin::Buffer::FileSingleChunk::FILE_PERMISSION, c1.permission
+      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
       assert_equal File.join(@bufdir, "fsb.foo_bar.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path
 
       c1.purge