fluentd 1.9.0-x86-mingw32 → 1.9.1-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c012a28dff140583d26c6cdf490dd3efdeedb5e502f3f6764796ff7415298385
-  data.tar.gz: e1bcdebb3998dfc297e60cddf9fa1c0db4079595169610b3088927a61c1d6d32
+  metadata.gz: cbfe3d4aeb87864958def88acc363c685f4d2ce227d3fddf390ec958a7552263
+  data.tar.gz: e906a7d5498b2ae4142dfd803912a6e8f8be8b274ea66a273f2c80e50d6b8051
 SHA512:
-  metadata.gz: dc4fb419d9a30830881e7dc5a563f7d19030008f5bac3452cb1549da6860aa58b28bec8b9bcb2e2c8b41a744ff1de1c713ff8235343e2154531ecbed09344bdd
-  data.tar.gz: 49553765cf43ab3094962306f4abfdf2c23b8da8695756702f2a1855b834130bf195a2ceaa39e48bb030680c480133129ff99125280840de3896208362765ef3
+  metadata.gz: 7b7d0e0413a70658a4acc138944f626bd011ebae3938f902cd0adbba3162964c2283a2e0b657591d3568ab438994ce7af35d4157efb28a95ad09dca3f3cae0f1
+  data.tar.gz: b41b6910c6a57c8cb4823fb86cc6d5c6c1f36a4a12ff7b6f591ccbdd2927f632f59635f2f7627ae3b594803b41eae179e9535587466c9697786eb0684495f141

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,7 @@
 <!--
 Thank you for contributing to Fluentd!
-Please provide the following information to help us make the most of your pull request:
+Your commits need to follow DCO: https://probot.github.io/apps/dco/
+And please provide the following information to help us make the most of your pull request:
 -->
 
 **Which issue(s) this PR fixes**: 

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # v1.9
 
+## Release v1.9.1 - 2020/01/31
+
+### Enhancement
+
+* http_server helper: Support HTTPS
+  https://github.com/fluent/fluentd/pull/2787
+* in_tail: Add `path_delimiter` to split with any char
+  https://github.com/fluent/fluentd/pull/2796
+* in_tail: Remove an entry from PositionaFile when it is unwatched
+  https://github.com/fluent/fluentd/pull/2803
+* out_http: Add warning for `retryable_response_code`
+  https://github.com/fluent/fluentd/pull/2809
+* parser_syslog: Add multiline RFC5424 support
+  https://github.com/fluent/fluentd/pull/2767
+* Add TLS module to unify TLS related code
+  https://github.com/fluent/fluentd/pull/2802
+
+### Bug fix
+
+* output: Add `EncodingError` to unrecoverable errors
+  https://github.com/fluent/fluentd/pull/2808
+* tls: Fix TLS version handling in secure mode
+  https://github.com/fluent/fluentd/pull/2802
+
 ## Release v1.9.0 - 2020/01/22
 
 ### New feature

data/Gemfile

@@ -2,8 +2,6 @@ source 'https://rubygems.org/'
 
 gemspec
 
-gem 'async-http', '~> 0.42'
-
 local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
 if File.exist?(local_gemfile)
   puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v`

data/appveyor.yml

@@ -5,11 +5,9 @@ version: '{build}'
 
 install:
   - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - IF %ridk%==0 "%devkit%\\devkitvars.bat"
   - ruby --version
   - gem --version
-  - IF %ridk%==1 ridk.cmd exec bundle install
-  - IF %ridk%==0 bundle install
+  - ridk.cmd exec bundle install
 build: off
 test_script:
   - bundle exec rake test
@@ -22,16 +20,9 @@ branches:
 # https://www.appveyor.com/docs/installed-software/#ruby
 environment:
   matrix:
-    - ruby_version: "23-x64"
-      devkit: C:\Ruby23-x64\DevKit
-      ridk: 0
-    - ruby_version: "23"
-      devkit: C:\Ruby23\DevKit
-      ridk: 0
+    - ruby_version: "26-x64"
+    - ruby_version: "26"
+    - ruby_version: "25-x64"
+    - ruby_version: "25"
     - ruby_version: "24-x64"
-      ridk: 1
     - ruby_version: "24"
-      ridk: 1
-matrix:
-  allow_failures:
-    - ruby_version: "21"

data/fluentd.gemspec

@@ -48,5 +48,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
-  gem.add_development_dependency("ext_monitor", [">= 0.1.1", "< 0.2"])
+  gem.add_development_dependency("ext_monitor", [">= 0.1.2", "< 0.2"])
+  gem.add_development_dependency("async-http")
 end

data/lib/fluent/config/section.rb

@@ -58,6 +58,10 @@ module Fluent
         @params
       end
 
+      def dup
+        Section.new(@params.dup, @corresponding_config_element.dup)
+      end
+
       def +(other)
         Section.new(self.to_h.merge(other.to_h))
       end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -210,7 +210,7 @@ module Fluent::Plugin
 
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
       api_handler = APIHandler.new(self)
-      create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
+      http_server_create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
         serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }
         serv.get('/api/plugins.json') { |req| api_handler.plugins_json(req) }
         serv.get('/api/config') { |req| api_handler.config_ltsv(req) }

data/lib/fluent/plugin/in_tail.rb

@@ -22,6 +22,7 @@ require 'fluent/event'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/parser_multiline'
 require 'fluent/variable_store'
+require 'fluent/plugin/in_tail/position_file'
 
 if Fluent.windows?
   require_relative 'file_wrapper'
@@ -35,6 +36,8 @@ module Fluent::Plugin
 
     helpers :timer, :event_loop, :parser, :compat_parameters
 
+    RESERVED_CHARS = ['/', '*', '%'].freeze
+
     class WatcherSetupError < StandardError
       def initialize(msg)
         @message = msg
@@ -58,6 +61,8 @@ module Fluent::Plugin
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
     config_param :path, :string
+    desc 'path delimiter used for spliting path config'
+    config_param :path_delimiter, :string, default: ','
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -113,9 +118,6 @@ module Fluent::Plugin
       unless parser_config
         raise Fluent::ConfigError, "<parse> section is required."
       end
-      unless parser_config["@type"]
-        raise Fluent::ConfigError, "parse/@type is required."
-      end
 
       (1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|
         parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
@@ -127,7 +129,12 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
-      @paths = @path.split(',').map {|path| path.strip }
+      if RESERVED_CHARS.include?(@path_delimiter)
+        rc = RESERVED_CHARS.join(', ')
+        raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
+      end
+
+      @paths = @path.split(@path_delimiter).map(&:strip)
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
@@ -391,7 +398,7 @@ module Fluent::Plugin
       tw.close if close_io
       flush_buffer(tw)
       if tw.unwatched && @pf
-        @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
+        @pf.unwatch(tw.path)
       end
     end
 
@@ -913,139 +920,5 @@ module Fluent::Plugin
         end
       end
     end
-
-    class PositionFile
-      UNWATCHED_POSITION = 0xffffffffffffffff
-
-      def initialize(file, file_mutex, map, last_pos)
-        @file = file
-        @file_mutex = file_mutex
-        @map = map
-        @last_pos = last_pos
-      end
-
-      def [](path)
-        if m = @map[path]
-          return m
-        end
-
-        @file_mutex.synchronize {
-          @file.pos = @last_pos
-          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
-          seek = @last_pos + path.bytesize + 1
-          @last_pos = @file.pos
-          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
-        }
-      end
-
-      def self.parse(file)
-        compact(file)
-
-        file_mutex = Mutex.new
-        map = {}
-        file.pos = 0
-        file.each_line {|line|
-          m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          unless m
-            $log.warn "Unparsable line in pos_file: #{line}"
-            next
-          end
-          path = m[1]
-          pos = m[2].to_i(16)
-          ino = m[3].to_i(16)
-          seek = file.pos - line.bytesize + path.bytesize + 1
-          map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
-        }
-        new(file, file_mutex, map, file.pos)
-      end
-
-      # Clean up unwatched file entries
-      def self.compact(file)
-        file.pos = 0
-        existent_entries = file.each_line.map { |line|
-          m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          unless m
-            $log.warn "Unparsable line in pos_file: #{line}"
-            next
-          end
-          path = m[1]
-          pos = m[2].to_i(16)
-          ino = m[3].to_i(16)
-          # 32bit inode converted to 64bit at this phase
-          pos == UNWATCHED_POSITION ? nil : ("%s\t%016x\t%016x\n" % [path, pos, ino])
-        }.compact
-
-        file.pos = 0
-        file.truncate(0)
-        file.write(existent_entries.join)
-      end
-    end
-
-    # pos               inode
-    # ffffffffffffffff\tffffffffffffffff\n
-    class FilePositionEntry
-      POS_SIZE = 16
-      INO_OFFSET = 17
-      INO_SIZE = 16
-      LN_OFFSET = 33
-      SIZE = 34
-
-      def initialize(file, file_mutex, seek, pos, inode)
-        @file = file
-        @file_mutex = file_mutex
-        @seek = seek
-        @pos = pos
-        @inode = inode
-      end
-
-      def update(ino, pos)
-        @file_mutex.synchronize {
-          @file.pos = @seek
-          @file.write "%016x\t%016x" % [pos, ino]
-        }
-        @pos = pos
-        @inode = ino
-      end
-
-      def update_pos(pos)
-        @file_mutex.synchronize {
-          @file.pos = @seek
-          @file.write "%016x" % pos
-        }
-        @pos = pos
-      end
-
-      def read_inode
-        @inode
-      end
-
-      def read_pos
-        @pos
-      end
-    end
-
-    class MemoryPositionEntry
-      def initialize
-        @pos = 0
-        @inode = 0
-      end
-
-      def update(ino, pos)
-        @inode = ino
-        @pos = pos
-      end
-
-      def update_pos(pos)
-        @pos = pos
-      end
-
-      def read_pos
-        @pos
-      end
-
-      def read_inode
-        @inode
-      end
-    end
   end
 end

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -0,0 +1,171 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/in_tail'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class PositionFile
+      UNWATCHED_POSITION = 0xffffffffffffffff
+      POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
+      POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
+
+      def initialize(file, file_mutex, map)
+        @file = file
+        @file_mutex = file_mutex
+        @map = map
+      end
+
+      def [](path)
+        if m = @map[path]
+          return m
+        end
+
+        @file_mutex.synchronize {
+          @file.seek(0, IO::SEEK_END)
+          seek = @file.pos + path.bytesize + 1
+          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
+          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+        }
+      end
+
+      def unwatch(path)
+        if (entry = @map.delete(path))
+          entry.update_pos(UNWATCHED_POSITION)
+        end
+      end
+
+      def self.parse(file)
+        compact(file)
+
+        file_mutex = Mutex.new
+        map = {}
+        file.pos = 0
+        file.each_line {|line|
+          m = POSITION_FILE_ENTRY_REGEX.match(line)
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
+          path = m[1]
+          pos = m[2].to_i(16)
+          ino = m[3].to_i(16)
+          seek = file.pos - line.bytesize + path.bytesize + 1
+          map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
+        }
+        new(file, file_mutex, map)
+      end
+
+      # Clean up unwatched file entries
+      def self.compact(file)
+        existent_entries = {}
+        file.pos = 0
+        file.each_line do |line|
+          m = POSITION_FILE_ENTRY_REGEX.match(line)
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
+          path = m[1]
+          pos = m[2].to_i(16)
+          ino = m[3].to_i(16)
+
+          if pos == UNWATCHED_POSITION
+            next
+          end
+
+          if existent_entries.include?(path)
+            $log.warn("#{path} already exists. use latest one: deleted #{existent_entries[path]}")
+          end
+
+          # 32bit inode converted to 64bit at this phase
+          existent_entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
+        end
+
+        file.pos = 0
+        file.truncate(0)
+        file.write(existent_entries.values.join)
+      end
+    end
+
+    # pos               inode
+    # ffffffffffffffff\tffffffffffffffff\n
+    class FilePositionEntry
+      POS_SIZE = 16
+      INO_OFFSET = 17
+      INO_SIZE = 16
+      LN_OFFSET = 33
+      SIZE = 34
+
+      def initialize(file, file_mutex, seek, pos, inode)
+        @file = file
+        @file_mutex = file_mutex
+        @seek = seek
+        @pos = pos
+        @inode = inode
+      end
+
+      def update(ino, pos)
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x\t%016x" % [pos, ino]
+        }
+        @pos = pos
+        @inode = ino
+      end
+
+      def update_pos(pos)
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x" % pos
+        }
+        @pos = pos
+      end
+
+      def read_inode
+        @inode
+      end
+
+      def read_pos
+        @pos
+      end
+    end
+
+    class MemoryPositionEntry
+      def initialize
+        @pos = 0
+        @inode = 0
+      end
+
+      def update(ino, pos)
+        @inode = ino
+        @pos = pos
+      end
+
+      def update_pos(pos)
+        @pos = pos
+      end
+
+      def read_pos
+        @pos
+      end
+
+      def read_inode
+        @inode
+      end
+    end
+  end
+end