fluentd 1.7.1 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bb8d41e15ede72bfbb90c06171c0eb4aa77d479cda1a8320f60e6166fbc27fc
-  data.tar.gz: d781e152f408a74e1cfed6b2c37f593db814d551d9f9179d1c5d15581218bf43
+  metadata.gz: 1ce449e77af62c25fcef41773c3717738ec443550becdde0ca666710739f7eae
+  data.tar.gz: 00d125b1c53ac99c8d2edf8b915909b91e07642e7fdad35154147a3b5a472416
 SHA512:
-  metadata.gz: 1f04e656acafae78bf4e2a770a5df16d549aa59ecd0a5883e97884b1de2dccc612108b11a1302b0655a2d56e3fb0b5f077da345441f34ce94a6bc5d31b3fac61
-  data.tar.gz: d16a3635f5c9607de95f66f3b836c44786abd7db9a4bb369b51f41862df0279bc3812962c633fe8ac8ad9f4439fe22b8aaf18ff7aa099dafe531956c0a33985e
+  metadata.gz: d78a18ab98f55b3568b0e401f87d80444d14b09b544fc4962a14bf821180c5f97a155332888961ce096adfa9ae6d01def4a154dc4a769939b6256846cf0b14b3
+  data.tar.gz: cc36a5137a9753d9888f6669d04dbfe943d35d3cb030f5718fa439f957e8658632e96246b219151b73f46a2ac74191a0f6e87f049aae1fe446b2e62fc3c4a274

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # v1.7
 
+## Release v1.7.2 - 2019/09/19
+
+### Enhancement
+
+* in_tcp: Add security/client to restrict access
+  https://github.com/fluent/fluentd/pull/2622
+
+### Bug fixes
+
+* buf_file/buf_file_single: fix to handle compress data during restart
+  https://github.com/fluent/fluentd/pull/2620
+* plugin: Use `__send__` to avoid conflict with user defined `send`
+  https://github.com/fluent/fluentd/pull/2614
+* buffer: reject invalid timekey at configure phase
+  https://github.com/fluent/fluentd/pull/2615
+
+
 ## Release v1.7.1 - 2019/09/08
 
 ### Enhancement

data/lib/fluent/plugin/buf_file.rb

@@ -146,7 +146,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
             handle_broken_files(path, mode, e)
             next

data/lib/fluent/plugin/buf_file_single.rb

@@ -167,7 +167,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path)
+            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
             chunk.restore_size(@chunk_format) if @calc_num_records
           rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
             handle_broken_files(path, mode, e)

data/lib/fluent/plugin/in_tcp.rb

@@ -41,6 +41,16 @@ module Fluent::Plugin
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
+    # in_forward like host/network restriction
+    config_section :security, required: false, multi: false do
+      config_section :client, param_name: :clients, required: true, multi: true do
+        desc 'The IP address or host name of the client'
+        config_param :host, :string, default: nil
+        desc 'Network address specification'
+        config_param :network, :string, default: nil
+      end
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
@@ -51,6 +61,33 @@ module Fluent::Plugin
       @_event_loop_blocking_timeout = @blocking_timeout
       @source_hostname_key ||= @source_host_key if @source_host_key
 
+      @nodes = nil
+      if @security
+        @nodes = []
+        @security.clients.each do |client|
+          if client.host && client.network
+            raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
+          end
+          if !client.host && !client.network
+            raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
+          end
+          source = nil
+          if client.host
+            begin
+              source = IPSocket.getaddress(client.host)
+            rescue SocketError
+              raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
+            end
+          end
+          source_addr = begin
+                          IPAddr.new(source || client.network)
+                        rescue ArgumentError
+                          raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
+                        end
+          @nodes.push(source_addr)
+        end
+      end
+
       @parser = parser_create(conf: parser_config)
     end
 
@@ -64,6 +101,11 @@ module Fluent::Plugin
       del_size = @delimiter.length
       if @_extract_enabled && @_extract_tag_key
         server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+          unless check_client(conn)
+            conn.close
+            next
+          end
+
           conn.buffer << data
           buf = conn.buffer
           pos = 0
@@ -89,6 +131,11 @@ module Fluent::Plugin
         end
       else
         server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+          unless check_client(conn)
+            conn.close
+            next
+          end
+
           conn.buffer << data
           buf = conn.buffer
           pos = 0
@@ -114,5 +161,20 @@ module Fluent::Plugin
         end
       end
     end
+
+    private
+
+    def check_client(conn)
+      if @nodes
+        remote_addr = conn.remote_addr
+        node = @nodes.find { |n| n.include?(remote_addr) rescue false }
+        unless node
+          log.warn "anonymous client '#{remote_addr}' denied"
+          return false
+        end
+      end
+
+      true
+    end
   end
 end

data/lib/fluent/plugin/multi_output.rb

@@ -94,7 +94,7 @@ module Fluent
         @outputs.each do |o|
           begin
             log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
-            o.send(method_name) unless o.send(checker_name)
+            o.__send__(method_name) unless o.__send__(checker_name)
           rescue Exception => e
             log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
             log.warn_backtrace

data/lib/fluent/plugin/output.rb

@@ -310,6 +310,9 @@ module Fluent
             Fluent::Timezone.validate!(@buffer_config.timekey_zone)
             @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
             @timekey = @buffer_config.timekey
+            if @timekey <= 0
+              raise Fluent::ConfigError, "timekey should be greater than 0. current timekey: #{@timekey}"
+            end
             @timekey_use_utc = @buffer_config.timekey_use_utc
             @offset = Fluent::Timezone.utc_offset(@timekey_zone)
             @calculate_offset = @offset.respond_to?(:call) ? @offset : nil

data/lib/fluent/plugin_helper/formatter.rb

@@ -103,7 +103,7 @@ module Fluent
       def formatter_operate(method_name, &block)
         @_formatters.each_pair do |usage, formatter|
           begin
-            formatter.send(method_name)
+            formatter.__send__(method_name)
             block.call(formatter) if block_given?
           rescue => e
             log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e

data/lib/fluent/plugin_helper/parser.rb

@@ -103,7 +103,7 @@ module Fluent
       def parser_operate(method_name, &block)
         @_parsers.each_pair do |usage, parser|
           begin
-            parser.send(method_name)
+            parser.__send__(method_name)
             block.call(parser) if block_given?
           rescue => e
             log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e

data/lib/fluent/plugin_helper/storage.rb

@@ -138,7 +138,7 @@ module Fluent
         @_storages.each_pair do |usage, s|
           begin
             block.call(s) if block_given?
-            s.storage.send(method_name)
+            s.storage.__send__(method_name)
           rescue => e
             log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
           end

data/lib/fluent/root_agent.rb

@@ -240,7 +240,7 @@ module Fluent
         lifecycle do |instance, kind|
           begin
             log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
-            instance.send(method) unless instance.send(checker)
+            instance.__send__(method) unless instance.__send__(checker)
           rescue Exception => e
             log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
             log.warn_backtrace
@@ -270,17 +270,17 @@ module Fluent
                 operation = "preparing shutdown" # for logging
                 log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
                 begin
-                  instance.send(:before_shutdown) unless instance.send(:before_shutdown?)
+                  instance.__send__(:before_shutdown) unless instance.__send__(:before_shutdown?)
                 rescue Exception => e
                   log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
                   log.warn_backtrace
                 end
                 operation = "shutting down"
                 log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
-                instance.send(:shutdown) unless instance.send(:shutdown?)
+                instance.__send__(:shutdown) unless instance.__send__(:shutdown?)
               else
                 log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
-                instance.send(method) unless instance.send(checker)
+                instance.__send__(method) unless instance.__send__(checker)
               end
             rescue Exception => e
               log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e

data/lib/fluent/system_config.rb

@@ -136,7 +136,7 @@ module Fluent
             supervisor_value = instance_variable_get("@#{param}")
             next if supervisor_value.nil? # it's not configured by command line options
 
-            system.send("#{param}=", supervisor_value)
+            system.__send__("#{param}=", supervisor_value)
           end
         end
       }
@@ -179,7 +179,7 @@ module Fluent
           @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
         end
         opts.each_pair do |key, value|
-          @_system_config.send(:"#{key.to_s}=", value)
+          @_system_config.__send__(:"#{key.to_s}=", value)
         end
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.7.1'
+  VERSION = '1.7.2'
 
 end

data/test/plugin/test_in_tcp.rb

@@ -163,6 +163,46 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal address, event[2]['addr']
   end
 
+  sub_test_case '<security>' do
+    test 'accept from allowed client' do
+      d = create_driver(CONFIG + %!
+        <security>
+          <client>
+            network 127.0.0.1
+          </client>
+        </security>
+      !)
+      d.run(expect_records: 1) do
+        create_tcp_socket('127.0.0.1', PORT) do |sock|
+          sock.send("hello\n", 0)
+        end
+      end
+
+      assert_equal 1, d.events.size
+      event = d.events[0]
+      assert_equal 'tcp', event[0]
+      assert_equal 'hello', event[2]['message']
+    end
+
+    test 'deny from disallowed client' do
+      d = create_driver(CONFIG + %!
+        <security>
+          <client>
+            network 200.0.0.0
+          </client>
+        </security>
+      !)
+      d.run(shutdown: false, expect_records: 1, timeout: 2) do
+        create_tcp_socket('127.0.0.1', PORT) do |sock|
+          sock.send("hello\n", 0)
+        end
+      end
+
+      assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
+      assert_equal 0, d.events.size
+    end
+  end
+
   sub_test_case '<extract>' do
     test 'extract tag from record field' do
       d = create_driver(BASE_CONFIG + %!

data/test/plugin/test_output.rb

@@ -868,6 +868,23 @@ class OutputTest < Test::Unit::TestCase
     end
   end
 
+  test 'raises an error if timekey is less than equal 0' do
+    i = create_output(:delayed)
+    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
+      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => nil })]))
+    end
+
+    i = create_output(:delayed)
+    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
+      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => 0 })]))
+    end
+
+    i = create_output(:delayed)
+    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: -1.0') do
+      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => -1 })]))
+    end
+  end
+
   sub_test_case 'sync output feature' do
     setup do
       @i = create_output(:sync)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-09 00:00:00.000000000 Z
+date: 2019-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack