fluentd 1.7.1 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bb8d41e15ede72bfbb90c06171c0eb4aa77d479cda1a8320f60e6166fbc27fc
-  data.tar.gz: d781e152f408a74e1cfed6b2c37f593db814d551d9f9179d1c5d15581218bf43
+  metadata.gz: 027c04b3446f864debe4b57849a0c30a2f3c24aac96f69e629d437a265ffafde
+  data.tar.gz: '0996f7d53774c1fc151b589b12570baf39f59b9edb39e761a40ea40f37908189'
 SHA512:
-  metadata.gz: 1f04e656acafae78bf4e2a770a5df16d549aa59ecd0a5883e97884b1de2dccc612108b11a1302b0655a2d56e3fb0b5f077da345441f34ce94a6bc5d31b3fac61
-  data.tar.gz: d16a3635f5c9607de95f66f3b836c44786abd7db9a4bb369b51f41862df0279bc3812962c633fe8ac8ad9f4439fe22b8aaf18ff7aa099dafe531956c0a33985e
+  metadata.gz: bf68902c8b6947a65956451d3601856d95f147428f3e3fc9b5f6a746ac15c02eaecb79a191c7811b605ca462310fac5849ce80167f37d1148c86f4f7e5598de4
+  data.tar.gz: 2c35f820be92a1e3741472c227c5fc8532aa06ef208001db981dd527d2bfa45c30d53888a4326e175e7db460980ade85a2f6b16bde6511940887d0786fdd9ad1

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # v1.7
 
+## Release v1.7.3 - 2019/10/01
+
+### Enhancement
+
+* in_syslog: Replace priority_key with severity_key
+  https://github.com/fluent/fluentd/pull/2636
+
+### Bug fixes
+
+* out_forward: Fix nil error after purge obsoleted sockets in socket cache
+  https://github.com/fluent/fluentd/pull/2635
+* fix typo in ChangeLog
+  https://github.com/fluent/fluentd/pull/2633
+
+## Release v1.7.2 - 2019/09/19
+
+### Enhancement
+
+* in_tcp: Add security/client to restrict access
+  https://github.com/fluent/fluentd/pull/2622
+
+### Bug fixes
+
+* buf_file/buf_file_single: fix to handle compress data during restart
+  https://github.com/fluent/fluentd/pull/2620
+* plugin: Use `__send__` to avoid conflict with user defined `send`
+  https://github.com/fluent/fluentd/pull/2614
+* buffer: reject invalid timekey at configure phase
+  https://github.com/fluent/fluentd/pull/2615
+
+
 ## Release v1.7.1 - 2019/09/08
 
 ### Enhancement
@@ -19,7 +50,7 @@
   https://github.com/fluent/fluentd/pull/2598
 * buf_file_single: fix duplicated path setting check
   https://github.com/fluent/fluentd/pull/2600
-* fix msgpack-ruby depedency to use recent feature
+* fix msgpack-ruby dependency to use recent feature
   https://github.com/fluent/fluentd/pull/2606
 
 
@@ -623,7 +654,7 @@
 
 ## Release v1.0.0 - 2017/12/6
 
-See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
+See [CNCF announcement](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
 
 ### New features / Enhancements
 
@@ -893,7 +924,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
   https://github.com/fluent/fluentd/pull/1492
 * parser: Allow escape sequence in Apache access log
   https://github.com/fluent/fluentd/pull/1479
-* config: Add actual value in the placholder error message
+* config: Add actual value in the placeholder error message
   https://github.com/fluent/fluentd/pull/1497
 * log: Add Fluent::Log#<< to support some SDKs
   https://github.com/fluent/fluentd/pull/1478
@@ -1210,7 +1241,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
   https://github.com/fluent/fluentd/pull/1067
 * filter_record_transformer: Fix to prevent overwriting reserved placeholder keys
   https://github.com/fluent/fluentd/pull/1176
-* Migrate some build-in plugins into v0.14 API
+* Migrate some built-in plugins into v0.14 API
   https://github.com/fluent/fluentd/pull/1149
   https://github.com/fluent/fluentd/pull/1151
 * Update dependencies

data/lib/fluent/plugin/buf_file.rb

@@ -146,7 +146,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
             handle_broken_files(path, mode, e)
             next

data/lib/fluent/plugin/buf_file_single.rb

@@ -167,7 +167,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path)
+            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
             chunk.restore_size(@chunk_format) if @calc_num_records
           rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
             handle_broken_files(path, mode, e)

data/lib/fluent/plugin/in_syslog.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
       23  => 'local7'
     }
 
-    PRIORITY_MAP = {
+    SEVERITY_MAP = {
       0  => 'emerg',
       1  => 'alert',
       2  => 'crit',
@@ -91,8 +91,8 @@ module Fluent::Plugin
     config_param :resolve_hostname, :bool, default: nil
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
-    desc 'The field name of the priority.'
-    config_param :priority_key, :string, default: nil
+    desc 'The field name of the severity.'
+    config_param :severity_key, :string, default: nil, alias: :priority_key
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
 
@@ -119,6 +119,10 @@ module Fluent::Plugin
 
       super
 
+      if conf.has_key?('priority_key')
+        log.warn "priority_key is deprecated. Use severity_key instead"
+      end
+
       @use_default = false
 
       @parser = parser_create
@@ -233,14 +237,14 @@ module Fluent::Plugin
 
         pri ||= record.delete('pri')
         facility = FACILITY_MAP[pri >> 3]
-        priority = PRIORITY_MAP[pri & 0b111]
+        severity = SEVERITY_MAP[pri & 0b111]
 
-        record[@priority_key] = priority if @priority_key
+        record[@severity_key] = severity if @severity_key
         record[@facility_key] = facility if @facility_key
         record[@source_address_key] = sock.remote_addr if @source_address_key
         record[@source_hostname_key] = sock.remote_host if @source_hostname_key
 
-        tag = "#{@tag}.#{facility}.#{priority}"
+        tag = "#{@tag}.#{facility}.#{severity}"
         emit(tag, time, record)
       end
     rescue => e

data/lib/fluent/plugin/in_tcp.rb

@@ -41,6 +41,16 @@ module Fluent::Plugin
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
+    # in_forward like host/network restriction
+    config_section :security, required: false, multi: false do
+      config_section :client, param_name: :clients, required: true, multi: true do
+        desc 'The IP address or host name of the client'
+        config_param :host, :string, default: nil
+        desc 'Network address specification'
+        config_param :network, :string, default: nil
+      end
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
@@ -51,6 +61,33 @@ module Fluent::Plugin
       @_event_loop_blocking_timeout = @blocking_timeout
       @source_hostname_key ||= @source_host_key if @source_host_key
 
+      @nodes = nil
+      if @security
+        @nodes = []
+        @security.clients.each do |client|
+          if client.host && client.network
+            raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
+          end
+          if !client.host && !client.network
+            raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
+          end
+          source = nil
+          if client.host
+            begin
+              source = IPSocket.getaddress(client.host)
+            rescue SocketError
+              raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
+            end
+          end
+          source_addr = begin
+                          IPAddr.new(source || client.network)
+                        rescue ArgumentError
+                          raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
+                        end
+          @nodes.push(source_addr)
+        end
+      end
+
       @parser = parser_create(conf: parser_config)
     end
 
@@ -64,6 +101,11 @@ module Fluent::Plugin
       del_size = @delimiter.length
       if @_extract_enabled && @_extract_tag_key
         server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+          unless check_client(conn)
+            conn.close
+            next
+          end
+
           conn.buffer << data
           buf = conn.buffer
           pos = 0
@@ -89,6 +131,11 @@ module Fluent::Plugin
         end
       else
         server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+          unless check_client(conn)
+            conn.close
+            next
+          end
+
           conn.buffer << data
           buf = conn.buffer
           pos = 0
@@ -114,5 +161,20 @@ module Fluent::Plugin
         end
       end
     end
+
+    private
+
+    def check_client(conn)
+      if @nodes
+        remote_addr = conn.remote_addr
+        node = @nodes.find { |n| n.include?(remote_addr) rescue false }
+        unless node
+          log.warn "anonymous client '#{remote_addr}' denied"
+          return false
+        end
+      end
+
+      true
+    end
   end
 end

data/lib/fluent/plugin/multi_output.rb

@@ -94,7 +94,7 @@ module Fluent
         @outputs.each do |o|
           begin
             log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
-            o.send(method_name) unless o.send(checker_name)
+            o.__send__(method_name) unless o.__send__(checker_name)
           rescue Exception => e
             log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
             log.warn_backtrace

data/lib/fluent/plugin/out_forward/socket_cache.rb

@@ -81,7 +81,9 @@ module Fluent::Plugin
               end
             end
           end
-          @available_sockets = @available_sockets.select { |_, v| !v.empty? }
+
+          # reuse same object (@available_sockets)
+          @available_sockets.reject! { |_, v| v.empty? }
 
           sockets += @inactive_sockets
           @inactive_sockets.clear

data/lib/fluent/plugin/output.rb

@@ -310,6 +310,9 @@ module Fluent
             Fluent::Timezone.validate!(@buffer_config.timekey_zone)
             @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
             @timekey = @buffer_config.timekey
+            if @timekey <= 0
+              raise Fluent::ConfigError, "timekey should be greater than 0. current timekey: #{@timekey}"
+            end
             @timekey_use_utc = @buffer_config.timekey_use_utc
             @offset = Fluent::Timezone.utc_offset(@timekey_zone)
             @calculate_offset = @offset.respond_to?(:call) ? @offset : nil

data/lib/fluent/plugin_helper/formatter.rb

@@ -103,7 +103,7 @@ module Fluent
       def formatter_operate(method_name, &block)
         @_formatters.each_pair do |usage, formatter|
           begin
-            formatter.send(method_name)
+            formatter.__send__(method_name)
             block.call(formatter) if block_given?
           rescue => e
             log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e

data/lib/fluent/plugin_helper/parser.rb

@@ -103,7 +103,7 @@ module Fluent
       def parser_operate(method_name, &block)
         @_parsers.each_pair do |usage, parser|
           begin
-            parser.send(method_name)
+            parser.__send__(method_name)
             block.call(parser) if block_given?
           rescue => e
             log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e

data/lib/fluent/plugin_helper/storage.rb

@@ -138,7 +138,7 @@ module Fluent
         @_storages.each_pair do |usage, s|
           begin
             block.call(s) if block_given?
-            s.storage.send(method_name)
+            s.storage.__send__(method_name)
           rescue => e
             log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
           end

data/lib/fluent/root_agent.rb

@@ -240,7 +240,7 @@ module Fluent
         lifecycle do |instance, kind|
           begin
             log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
-            instance.send(method) unless instance.send(checker)
+            instance.__send__(method) unless instance.__send__(checker)
           rescue Exception => e
             log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
             log.warn_backtrace
@@ -270,17 +270,17 @@ module Fluent
                 operation = "preparing shutdown" # for logging
                 log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
                 begin
-                  instance.send(:before_shutdown) unless instance.send(:before_shutdown?)
+                  instance.__send__(:before_shutdown) unless instance.__send__(:before_shutdown?)
                 rescue Exception => e
                   log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
                   log.warn_backtrace
                 end
                 operation = "shutting down"
                 log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
-                instance.send(:shutdown) unless instance.send(:shutdown?)
+                instance.__send__(:shutdown) unless instance.__send__(:shutdown?)
               else
                 log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
-                instance.send(method) unless instance.send(checker)
+                instance.__send__(method) unless instance.__send__(checker)
               end
             rescue Exception => e
               log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e

data/lib/fluent/system_config.rb

@@ -136,7 +136,7 @@ module Fluent
             supervisor_value = instance_variable_get("@#{param}")
             next if supervisor_value.nil? # it's not configured by command line options
 
-            system.send("#{param}=", supervisor_value)
+            system.__send__("#{param}=", supervisor_value)
           end
         end
       }
@@ -179,7 +179,7 @@ module Fluent
           @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
         end
         opts.each_pair do |key, value|
-          @_system_config.send(:"#{key.to_s}=", value)
+          @_system_config.__send__(:"#{key.to_s}=", value)
         end
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.7.1'
+  VERSION = '1.7.3'
 
 end

data/test/plugin/out_forward/test_socket_cache.rb

@@ -5,7 +5,7 @@ require 'timecop'
 
 class SocketCacheTest < Test::Unit::TestCase
   sub_test_case 'checkout_or' do
-    test 'when gived key does not exist' do
+    test 'when given key does not exist' do
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
       sock = mock!.open { 'socket' }.subject
       assert_equal('socket', c.checkout_or('key') { sock.open })
@@ -38,6 +38,16 @@ class SocketCacheTest < Test::Unit::TestCase
       sock = mock!.open { new_sock }.subject
       assert_equal(new_sock, c.checkout_or('key') { sock.open })
     end
+
+    test 'reuse same hash object after calling purge_obsolete_socks' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      c.checkout_or('key') { 'socket' }
+      c.purge_obsolete_socks
+
+      assert_nothing_raised(NoMethodError) do
+        c.checkout_or('key') { 'new socket' }
+      end
+    end
   end
 
   sub_test_case 'checkin' do

data/test/plugin/test_buffer_file_chunk.rb

@@ -495,7 +495,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
     end
   end
 
-  test 'ensure to remove metadata file if #write_metadata raise an error becuase of disk full' do
+  test 'ensure to remove metadata file if #write_metadata raise an error because of disk full' do
     chunk_path = File.join(@chunkdir, 'test.*.log')
     stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id
 

data/test/plugin/test_in_syslog.rb

@@ -217,11 +217,15 @@ EOS
     compare_test_result(d.events, tests, {host: host})
   end
 
-  def test_msg_size_with_priority_key
-    d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
+  data(
+    severity_key: 'severity_key',
+    priority_key: 'priority_key',
+  )
+  def test_msg_size_with_severity_key(param_name)
+    d = create_driver([CONFIG, "#{param_name} severity"].join("\n"))
     tests = create_test_case
 
-    priority = 'info'
+    severity = 'info'
     d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
@@ -231,7 +235,7 @@ EOS
     end
 
     assert(d.events.size > 0)
-    compare_test_result(d.events, tests, {priority: priority})
+    compare_test_result(d.events, tests, {severity: severity})
   end
 
   def test_msg_size_with_facility_key
@@ -311,7 +315,7 @@ EOS
       assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
       assert_equal(options[:address], events[i][2]['source_address']) if options[:address]
       assert_equal(options[:hostname], events[i][2]['source_hostname']) if options[:hostname]
-      assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
+      assert_equal(options[:severity], events[i][2]['severity']) if options[:severity]
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end

data/test/plugin/test_in_tcp.rb

@@ -163,6 +163,46 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal address, event[2]['addr']
   end
 
+  sub_test_case '<security>' do
+    test 'accept from allowed client' do
+      d = create_driver(CONFIG + %!
+        <security>
+          <client>
+            network 127.0.0.1
+          </client>
+        </security>
+      !)
+      d.run(expect_records: 1) do
+        create_tcp_socket('127.0.0.1', PORT) do |sock|
+          sock.send("hello\n", 0)
+        end
+      end
+
+      assert_equal 1, d.events.size
+      event = d.events[0]
+      assert_equal 'tcp', event[0]
+      assert_equal 'hello', event[2]['message']
+    end
+
+    test 'deny from disallowed client' do
+      d = create_driver(CONFIG + %!
+        <security>
+          <client>
+            network 200.0.0.0
+          </client>
+        </security>
+      !)
+      d.run(shutdown: false, expect_records: 1, timeout: 2) do
+        create_tcp_socket('127.0.0.1', PORT) do |sock|
+          sock.send("hello\n", 0)
+        end
+      end
+
+      assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
+      assert_equal 0, d.events.size
+    end
+  end
+
   sub_test_case '<extract>' do
     test 'extract tag from record field' do
       d = create_driver(BASE_CONFIG + %!

data/test/plugin/test_output.rb

@@ -868,6 +868,23 @@ class OutputTest < Test::Unit::TestCase
     end
   end
 
+  test 'raises an error if timekey is less than equal 0' do
+    i = create_output(:delayed)
+    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
+      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => nil })]))
+    end
+
+    i = create_output(:delayed)
+    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
+      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => 0 })]))
+    end
+
+    i = create_output(:delayed)
+    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: -1.0') do
+      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => -1 })]))
+    end
+  end
+
   sub_test_case 'sync output feature' do
     setup do
       @i = create_output(:sync)

data/test/plugin/test_output_as_buffered.rb

@@ -1112,7 +1112,7 @@ class BufferedOutputTest < Test::Unit::TestCase
         assert_equal(0, @i.write_count)
 
         Timecop.freeze(Time.parse('2019-02-09 00:00:12 +0900'))
-        # wirte should be called in few seconds since
+        # write should be called in few seconds since
         # running interval of enque thread is timekey_wait / 11.0.
         waiting(5){ sleep 0.1 until @i.write_count == 1 }
       end

data/test/test_root_agent.rb

@@ -710,7 +710,7 @@ EOC
       end
     end
 
-    test 'raises configuration error for worker id collisions on multi workers syntax when multi avaliable worker_ids are left' do
+    test 'raises configuration error for worker id collisions on multi workers syntax when multi available worker_ids are left' do
       errmsg = "specified worker_id<1> collisions is detected on <worker> directive. Available worker id(s): [2, 3]"
       assert_raise Fluent::ConfigError.new(errmsg) do
         conf = <<-EOC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-09 00:00:00.000000000 Z
+date: 2019-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack