fluentd 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 784878757e32d7abd7c3bed9bcb4f081524502b0
-  data.tar.gz: 89205de62e14d5b3210888a40656df87c6bc8d15
+SHA256:
+  metadata.gz: ee967542a2be7009af606bfb3b78f0a8a9481d4649943e36cb6274629b235fde
+  data.tar.gz: ceba4cc28ba327a073df7520266570f5d59b43fb1bdb7952aa4c24297d4ce3ce
 SHA512:
-  metadata.gz: fcbe5512f4ce1ffef0af5948dd4a6a1b81095f2d3032dee86692db4eb1164f13a2710786562980d2d44f3d289f9120d53c222b4ca796eae58fbc15e0354a657a
-  data.tar.gz: 918b3cad26663a13c5ccba5e4fea2494124d793e5744e1c4e7e2b2d844a758638a3b78843e3718af7e1c95ee8420c60c06b469531dc0bba5151504817e970c64
+  metadata.gz: 0ac3ac045228b9d883c00248e44202f97975ea4ad320b868823e8fc15d0f68f1007012494777760fe3aff695290321776b54b25ed494ae00e190123e928f5f9e
+  data.tar.gz: f9360951581699824d0eef3646c34dac7b555555e1f578cb756605ca9fef8c2fe66c3443195e45d1ff57fae08e9d49c4c90d404b48c91d53494ff31181ba4feb

data/CHANGELOG.md

@@ -1,4 +1,23 @@
-# v1.1
+# v1.2
+
+## Release v1.2.1 - 2018/05/23
+
+### Enhancements
+
+* Counter: Add wait API to client
+  https://github.com/fluent/fluentd/pull/1997
+
+### Bug fixes
+
+* in_tcp/in_udp: Fix source_hostname_key to set hostname correctly
+  https://github.com/fluent/fluentd/pull/1976
+* in_monitor_agent: Fix buffer_total_queued_size calculation
+  https://github.com/fluent/fluentd/pull/1990
+* out_file: Temporal fix for broken gzipped files with gzip and append
+  https://github.com/fluent/fluentd/pull/1995
+* test: Fix unstable backup test
+  https://github.com/fluent/fluentd/pull/1979
+* gemspec: Remove deprecated has_rdoc
 
 ## Release v1.2.0 - 2018/04/30
 
@@ -8,7 +27,7 @@
   https://github.com/fluent/fluentd/pull/1857
 * output: Backup for broken chunks
   https://github.com/fluent/fluentd/pull/1952
-* filter_grep: Support `<and>` and `<or>` support
+* filter_grep: Support for `<and>` and `<or>` sections
   https://github.com/fluent/fluentd/pull/1897
 * config: Support `regexp` type in configuration parameter
   https://github.com/fluent/fluentd/pull/1927
@@ -28,6 +47,8 @@
   https://github.com/fluent/fluentd/pull/1942
 * output: Buffer chunk's unique id should be formatted as hex in the log
 
+# v1.1
+
 ## Release v1.1.3 - 2018/04/03
 
 ### Enhancements

data/fluentd.gemspec

@@ -14,7 +14,6 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.has_rdoc = false
   gem.license = "Apache-2.0"
 
   gem.required_ruby_version = '>= 2.1'

data/lib/fluent/counter/client.rb

@@ -16,6 +16,7 @@
 
 require 'cool.io'
 require 'fluent/counter/base_socket'
+require 'fluent/counter/error'
 require 'timeout'
 
 module Fluent
@@ -59,7 +60,7 @@ module Fluent
       def establish(scope)
         scope = Timeout.timeout(@timeout) {
           response = send_request('establish', nil, [scope])
-          raise response.errors.first if response.errors?
+          Fluent::Counter.raise_error(response.errors.first) if response.errors?
           data = response.data
           data.first
         }
@@ -274,6 +275,14 @@ module Fluent
         @result
       end
 
+      def wait
+        res = get
+        if res.error?
+          Fluent::Counter.raise_error(res.errors.first)
+        end
+        res
+      end
+
       private
 
       def join

data/lib/fluent/counter/error.rb

@@ -61,5 +61,26 @@ module Fluent
         'internal_server_error'
       end
     end
+
+    def raise_error(response)
+      msg = response['message']
+      case response['code']
+      when 'invalid_params'
+        raise InvalidParams.new(msg)
+      when 'unknown_key'
+        raise UnknownKey.new(msg)
+      when 'parse_error'
+        raise ParseError.new(msg)
+      when 'invalid_request'
+        raise InvalidRequest.new(msg)
+      when 'method_not_found'
+        raise MethodNotFound.new(msg)
+      when 'internal_server_error'
+        raise InternalServerError.new(msg)
+      else
+        raise "Unknown code: #{response['code']}"
+      end
+    end
+    module_function :raise_error
   end
 end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -279,7 +279,7 @@ module Fluent::Plugin
     MONITOR_INFO = {
       'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
       'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
-      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size },
+      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
       'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
     }
 

data/lib/fluent/plugin/in_tcp.rb

@@ -55,7 +55,7 @@ module Fluent::Plugin
     def start
       super
 
-      server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
+      server_create(:in_tcp_server, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
         conn.buffer << data
         begin
           pos = 0

data/lib/fluent/plugin/in_udp.rb

@@ -63,7 +63,7 @@ module Fluent::Plugin
       super
 
       log.info "listening udp socket", bind: @bind, port: @port
-      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, resolve_name: !!@source_hostname_key, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
         data.chomp! if @remove_newline
         begin
           @parser.parse(data) do |time, record|

data/lib/fluent/plugin/out_file.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'zlib'
 require 'time'
+require 'tempfile'
 
 require 'fluent/plugin/output'
 require 'fluent/config/error'
@@ -213,10 +214,28 @@ module Fluent::Plugin
     end
 
     def write_gzip_with_compression(path, chunk)
-      File.open(path, "ab", @file_perm) do |f|
-        gz = Zlib::GzipWriter.new(f)
-        chunk.write_to(gz, compressed: :text)
-        gz.close
+      if @append
+        # This code will be removed after zlib/multithread bug is fixed.
+        # Use Tempfile to avoid broken gzip files: https://github.com/fluent/fluentd/issues/1903
+        Tempfile.create('out_file-gzip-append') { |temp|
+          begin
+            writer = Zlib::GzipWriter.new(temp)
+            chunk.write_to(writer, compressed: :text)
+          ensure
+            writer.finish # avoid zlib finalizer warning
+          end
+          temp.rewind
+
+          File.open(path, "ab", @file_perm) do |f|
+            IO.copy_stream(temp, f)
+          end
+        }
+      else
+        File.open(path, "ab", @file_perm) do |f|
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz, compressed: :text)
+          gz.close
+        end
       end
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.2.0'
+  VERSION = '1.2.1'
 
 end

data/test/counter/test_client.rb

@@ -151,6 +151,10 @@ class CounterClientTest < ::Test::Unit::TestCase
 
       assert_empty response.data
       assert_equal expected_error, errors
+
+      assert_raise {
+        @client.init(param).wait
+      }
     end
 
     test 'return an existing value when passed key already exists and ignore option is true' do

data/test/plugin/test_in_monitor_agent.rb

@@ -455,7 +455,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       d.instance.start
       expected_test_out_fail_write_response = {
           "buffer_queue_length" => 1,
-          "buffer_total_queued_size" => 0,
+          "buffer_total_queued_size" => 40,
           "output_plugin" => true,
           "plugin_category" => "output",
           "plugin_id" => "test_out_fail_write",

data/test/plugin/test_in_tcp.rb

@@ -115,4 +115,25 @@ class TcpInputTest < Test::Unit::TestCase
       assert_equal expected_record, d.events[i][2]
     end
   end
+
+  test 'source_hostname_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_hostname_key host
+    !)
+    hostname = nil
+    d.run(expect_records: 1) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        sock.do_not_reverse_lookup = false
+        hostname = sock.peeraddr[2]
+        sock.send("test\n", 0)
+      end
+    end
+
+    assert_equal 1, d.events.size
+    event = d.events[0]
+    assert_equal "tcp", event[0]
+    assert event[1].is_a?(Fluent::EventTime)
+    assert_equal hostname, event[2]['host']
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -31,6 +31,7 @@ class UdpInputTest < Test::Unit::TestCase
         else
           UDPSocket.new(Socket::AF_INET6)
         end
+    u.do_not_reverse_lookup = false
     u.connect(host, port)
     if block_given?
       begin
@@ -174,6 +175,26 @@ class UdpInputTest < Test::Unit::TestCase
     end
   end
 
+  test 'source_hostname_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_hostname_key host
+    !)
+    hostname = nil
+    d.run(expect_records: 1) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        u.send("test", 0)
+        hostname = u.peeraddr[2]
+      end
+    end
+
+    expected = {'message' => 'test'}
+    assert_equal 1, d.events.size
+    assert_equal "udp", d.events[0][0]
+    assert d.events[0][1].is_a?(Fluent::EventTime)
+    assert_equal hostname, d.events[0][2]['host']
+  end
+
   test 'receive_buffer_size' do
     # doesn't check exact value because it depends on platform and condition
 

data/test/plugin/test_output_as_buffered_backup.rb

@@ -169,6 +169,14 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
       @i.flush_thread_wakeup
     end
 
+    def wait_flush(target_file)
+      waiting(5) {
+        target_dir = File.join(File.dirname(target_file), "*")
+        while Dir.glob(target_dir).size.zero?
+        end
+      }
+    end
+
     test 'backup chunk without secondary' do
       Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
         id = 'backup_test'
@@ -186,6 +194,7 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
         flush_chunks
 
         target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
         assert_true File.exist?(target)
         logs = @i.log.out.logs
         assert { logs.any? { |l| l.include?("got unrecoverable error in primary and no secondary") } }
@@ -210,6 +219,7 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
         flush_chunks
 
         target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
         assert_true File.exist?(target)
         logs = @i.log.out.logs
         assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary type is same as primary") } }
@@ -237,6 +247,7 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
         flush_chunks
 
         target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
         assert_true File.exist?(target)
         logs = @i.log.out.logs
         assert { logs.any? { |l| l.include?("got unrecoverable error in primary. Skip retry and flush chunk to secondary") } }
@@ -262,6 +273,7 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
         flush_chunks
 
         target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
         assert_true File.exist?(target)
         logs = @i.log.out.logs
         assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-30 00:00:00.000000000 Z
+date: 2018-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -769,7 +769,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector