fluentd 1.2.5 → 1.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a8c8b50f9093742b22d626d6051608b7d9cce711
-  data.tar.gz: 3e3ad4d8f6af353c97d19856303e0f2ff082a7ac
+  metadata.gz: 8429bf1a0e35278c81b67e3d517568038a67fb2c
+  data.tar.gz: ba5260ae4f5db3933aebbf2ba598f0c03d9ab878
 SHA512:
-  metadata.gz: becc9444e4bbd8205e4bc7dea1414a5e29d4bb95f84b5708b01c9008369bdbf4a563fbe40e6230c328371fd8765177e2db59fd11ee7bd900e3cc0a58903768d3
-  data.tar.gz: 62dfa273cb9754d521ff4f652d8390d9315bf1a2db519a7bc590ec61a56ca320e0ab2576dddf3ae7ee133d94779cfc665f026c1571c1d88f38767c68843abced
+  metadata.gz: 575ca7960c2f0560d103001eecbbaf43763394c238280a4aebc4b8ab61336a069d29917c8dca25be95a2b78557c80e172e3dc0834ed8d165046bc87c822be426
+  data.tar.gz: be8459ae73c470d11848afc2f5cdc5e4f57a9a74606715e3491a7660e1fd4e362b899c5c7ccbe72504c74bcf1c4d818c6cbb00db37707529e6b30dd4d98c13cf

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # v1.2
 
+## Release v1.2.6 - 2018/10/03
+
+### Enhancements
+
+* output: Add `disable_chunk_backup` for ignore broken chunks.
+  https://github.com/fluent/fluentd/pull/2117
+* parser_syslog: Improve regexp for RFC5424
+  https://github.com/fluent/fluentd/pull/2141
+* in_http: Allow specifying the wildcard '*' as the CORS domain
+  https://github.com/fluent/fluentd/pull/2139
+
+### Bug fixes
+
+* in_tail: Prevent thread switching in the interval between seek and read/write operations to pos_file
+  https://github.com/fluent/fluentd/pull/2118
+* parser: Handle LoadError properly for oj
+  https://github.com/fluent/fluentd/pull/2140
+
 ## Release v1.2.5 - 2018/08/22
 
 ### Bug fixes

data/lib/fluent/plugin/in_http.rb

@@ -375,7 +375,7 @@ module Fluent::Plugin
         # For every incoming request, we check if we have some CORS
         # restrictions and white listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?(@origin)
+          unless @cors_allow_origins.include?('*') or @cors_allow_origins.include?(@origin)
             send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
             return
           end
@@ -422,7 +422,14 @@ module Fluent::Plugin
         code, header, body = *@callback.call(path_info, params)
         body = body.to_s
 
-        header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
+        unless @cors_allow_origins.nil?
+          if @cors_allow_origins.include?('*')
+            header['Access-Control-Allow-Origin'] = '*'
+          elsif @cors_allow_origins.include?(@origin)
+            header['Access-Control-Allow-Origin'] = @origin
+          end
+        end
+
         if @keep_alive
           header['Connection'] = 'Keep-Alive'
           send_response(code, header, body)

data/lib/fluent/plugin/in_tail.rb

@@ -869,8 +869,9 @@ module Fluent::Plugin
     class PositionFile
       UNWATCHED_POSITION = 0xffffffffffffffff
 
-      def initialize(file, map, last_pos)
+      def initialize(file, file_mutex, map, last_pos)
         @file = file
+        @file_mutex = file_mutex
         @map = map
         @last_pos = last_pos
       end
@@ -880,31 +881,34 @@ module Fluent::Plugin
           return m
         end
 
-        @file.pos = @last_pos
-        @file.write path
-        @file.write "\t"
-        seek = @file.pos
-        @file.write "0000000000000000\t0000000000000000\n"
-        @last_pos = @file.pos
-
-        @map[path] = FilePositionEntry.new(@file, seek, 0, 0)
+        @file_mutex.synchronize {
+          @file.pos = @last_pos
+          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
+          seek = @last_pos + path.bytesize + 1
+          @last_pos = @file.pos
+          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+        }
       end
 
       def self.parse(file)
         compact(file)
 
+        file_mutex = Mutex.new
         map = {}
         file.pos = 0
         file.each_line {|line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
           path = m[1]
           pos = m[2].to_i(16)
           ino = m[3].to_i(16)
           seek = file.pos - line.bytesize + path.bytesize + 1
-          map[path] = FilePositionEntry.new(file, seek, pos, ino)
+          map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
         }
-        new(file, map, file.pos)
+        new(file, file_mutex, map, file.pos)
       end
 
       # Clean up unwatched file entries
@@ -912,7 +916,10 @@ module Fluent::Plugin
         file.pos = 0
         existent_entries = file.each_line.map { |line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
           path = m[1]
           pos = m[2].to_i(16)
           ino = m[3].to_i(16)
@@ -935,23 +942,28 @@ module Fluent::Plugin
       LN_OFFSET = 33
       SIZE = 34
 
-      def initialize(file, seek, pos, inode)
+      def initialize(file, file_mutex, seek, pos, inode)
         @file = file
+        @file_mutex = file_mutex
         @seek = seek
         @pos = pos 
         @inode = inode
       end
 
       def update(ino, pos)
-        @file.pos = @seek
-        @file.write "%016x\t%016x" % [pos, ino]
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x\t%016x" % [pos, ino]
+        }
         @pos = pos
         @inode = ino
       end
 
       def update_pos(pos)
-        @file.pos = @seek
-        @file.write "%016x" % pos
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x" % pos
+        }
         @pos = pos
       end
 

data/lib/fluent/plugin/output.rb

@@ -96,6 +96,7 @@ module Fluent
         config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
 
         config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
+        config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
       end
 
       config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
@@ -1161,17 +1162,21 @@ module Fluent
       end
 
       def backup_chunk(chunk, using_secondary, delayed_commit)
-        unique_id = dump_unique_id_hex(chunk.unique_id)
-        safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
-        backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
-        backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
-        backup_dir = File.dirname(backup_file)
-
-        log.warn "bad chunk is moved to #{backup_file}"
-        FileUtils.mkdir_p(backup_dir) unless Dir.exist?(backup_dir)
-        File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
-          chunk.write_to(f)
-        }
+        if @buffer_config.disable_chunk_backup
+          log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
+        else
+          unique_id = dump_unique_id_hex(chunk.unique_id)
+          safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+          backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
+          backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
+          backup_dir = File.dirname(backup_file)
+
+          log.warn "bad chunk is moved to #{backup_file}"
+          FileUtils.mkdir_p(backup_dir) unless Dir.exist?(backup_dir)
+          File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
+            chunk.write_to(f)
+          }
+        end
         commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
       end
 

data/lib/fluent/plugin/parser_json.rb

@@ -52,9 +52,15 @@ module Fluent
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
-      rescue LoadError
+      rescue LoadError => ex
         name = :yajl
-        log.info "Oj is not installed, and failing back to Yajl for json parser" if log
+        if log
+          if /\boj\z/ =~ ex.message
+            log.info "Oj is not installed, and failing back to Yajl for json parser"
+          else
+            log.warn ex.message
+          end
+        end
         retry
       end
 

data/lib/fluent/plugin/parser_syslog.rb

@@ -27,8 +27,8 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
-      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.2.5'
+  VERSION = '1.2.6'
 
 end

data/test/command/test_plugin_config_formatter.rb

@@ -155,6 +155,7 @@ slow_flush_log_threshold: float: (20.0)
  retry_exponential_backoff_base: float: (2)
  retry_max_interval: time: (nil)
  retry_randomize: bool: (true)
+ disable_chunk_backup: bool: (false)
 <secondary>: optional, single
  @type: string: (nil)
  <buffer>: optional, single

data/test/plugin/test_in_http.rb

@@ -604,6 +604,26 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_cors_allowed_wildcard
+    d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    events = [
+      ["tag1", time, {"a"=>1}],
+    ]
+
+    d.run do
+      events.each do |tag, time, record|
+        headers = {"Origin" => "http://foo.com"}
+
+        res = post("/#{tag}", {"json" => record.to_json, "time" => time.to_i}, headers)
+
+        assert_equal "200", res.code
+        assert_equal "*", res["Access-Control-Allow-Origin"]
+      end
+    end
+  end
+
   def test_content_encoding_gzip
     d = create_driver
 

data/test/plugin/test_output_as_buffered_backup.rb

@@ -279,5 +279,29 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
         assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }
       end
     end
+
+    test 'chunk is thrown away when disable_chunk_backup is true' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+          'disable_chunk_backup' => true
+        }
+        chunk_id = nil
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_false File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("disable_chunk_backup is true") } }
+      end
+    end
   end
 end

data/test/plugin/test_parser_syslog.rb

@@ -118,6 +118,23 @@ class SyslogParserTest < ::Test::Unit::TestCase
                    @parser.instance.patterns['format'])
     end
 
+    def test_parse_with_rfc5424_empty_message_and_without_priority
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_nil record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
+                   @parser.instance.patterns['format'])
+    end
+
     def test_parse_with_rfc5424_message_without_time_format
       @parser.configure(
                         'message_format' => 'rfc5424',
@@ -133,6 +150,21 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
+    def test_parse_with_rfc5424_message_with_priority_and_pid
+      @parser.configure(
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - -  peer 192.168.0.5:50123 is not from a LAN'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2018-09-26T15:54:26.620412+0900", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "1298", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal " peer 192.168.0.5:50123 is not from a LAN", record["message"]
+      end
+    end
+
     def test_parse_with_rfc5424_structured_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
@@ -142,13 +174,64 @@ class SyslogParserTest < ::Test::Unit::TestCase
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-assert_equal "11111", record["pid"]
+        assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
     end
+
+    def test_parse_with_rfc5424_multiple_structured_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_message_includes_right_bracket
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd]!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_empty_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_nil record["message"]
+      end
+    end
   end
 
   class TestAutoRegexp < self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.2.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-23 00:00:00.000000000 Z
+date: 2018-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack