fluentd 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e20dab78cc15d81c32ba9cd70fd5a806c6234a73
-  data.tar.gz: 2c133e7665781dcc0d97da8459489ce3edf46f8f
+  metadata.gz: 3a26c51f262d85284f6ec0144fcab0088682efa9
+  data.tar.gz: 74f6bf4722ca406b51e678b7c0037f8f0c2e0a16
 SHA512:
-  metadata.gz: 3d3775f9459c4c5ee8747c39596d2750e51ae6e0bab04958c6162d1faad4a6afd7deb2bcce27ed14b2f1eb925666554d444506cc30408a5db898f29aab3c6258
-  data.tar.gz: c67fcc90f8f3717b73f4d3d78d5899881c688d52d3a39d4b0e112412092973a65d86ca7e9930326f894f3e896013afa42971532db895d67ecbc6e48c5b12180b
+  metadata.gz: 5de45a297b116281323d7c1d6cba55e46300c77025e4fa5f59de6be27bb21606448c3c2c86142c69e7b5ba4da0cbadb2b3f0acc23af6d3046c55521080559767
+  data.tar.gz: e9b1ddc803810bfd8b98fd2b3d5378bc1f50802b72e7343772b5bef229d8982d6d5e5a2aabf075c1ebe6d8d7a24c7931a3e95833f5e731943cf4d98f2bc66abc

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # v1.1
 
+## Release v1.1.3 - 2018/04/03
+
+### Enhancements
+
+* output: Support negative index for tag placeholders
+  https://github.com/fluent/fluentd/pull/1908
+* buffer: Add queued_chunks_limit_size to control the number of queued chunks
+  https://github.com/fluent/fluentd/pull/1916
+* time: Make Fluent::EventTime human readable for inspect
+  https://github.com/fluent/fluentd/pull/1915
+
+### Bug fixes
+
+* output: Delete empty queued_num field after purging chunks
+  https://github.com/fluent/fluentd/pull/1919
+* fluent-debug: Fix usage message of fluent-debug command
+  https://github.com/fluent/fluentd/pull/1920
+* out_forward: The node should be disabled when TLS socket for ack returns an error
+  https://github.com/fluent/fluentd/pull/1925
+
 ## Release v1.1.2 - 2018/03/18
 
 ### Enhancements

data/CONTRIBUTING.md

@@ -36,6 +36,7 @@ Here are some things that would increase a chance that your patch is accepted:
 * Write tests.
 * Run tests before send Pull Request by `bundle exec rake test`
 * Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+  * Fluentd repositories needs [DCO](https://github.com/apps/dco) on PR. Please add `Signed-off-by` to the commit(See DCO link for more detail).
 
 There are some patches which are hard to write tests, e.g. process handling, concurrency issue or etc.
 In such case, please don't hesitate to submit a Pull Request.

data/lib/fluent/command/debug.rb

@@ -91,9 +91,9 @@ include Fluent
 
 puts "Connected to #{uri}."
 puts "Usage:"
-puts "    Fluent::Engine.match('some.tag').output  : get an output plugin instance"
-puts "    Fluent::Engine.sources[i]                : get input plugin instances"
-puts "    Fluent::Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
+puts "    Fluent::Engine.root_agent.event_router.match('some.tag') : get an output plugin instance"
+puts "    Fluent::Engine.root_agent.inputs[i]                      : get input plugin instances"
+puts "    Fluent::Plugin::OUTPUT_REGISTRY.lookup(name)             : load output plugin class (use this if you get DRb::DRbUnknown)"
 puts ""
 
 Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)

data/lib/fluent/plugin/buffer.rb

@@ -55,6 +55,9 @@ module Fluent
       # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
       config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
 
+      desc 'The max number of queued chunks.'
+      config_param :queued_chunks_limit_size, :integer, default: nil
+
       desc 'Compress buffered data.'
       config_param :compress, :enum, list: [:text, :gzip], default: :text
 
@@ -367,6 +370,10 @@ module Fluent
         end
       end
 
+      def queue_full?
+        @queued_chunks_limit_size && (synchronize { @queue.size } >= @queued_chunks_limit_size)
+      end
+
       def queued_records
         synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }
       end
@@ -421,11 +428,13 @@ module Fluent
         end
       end
 
-      def enqueue_all
+      # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
+      def enqueue_all(force_enqueue = false)
         log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
 
         if block_given?
           synchronize{ @stage.keys }.each do |metadata|
+            return if !force_enqueue && queue_full?
             # NOTE: The following line might cause data race depending on Ruby implementations except CRuby
             # cf. https://github.com/fluent/fluentd/pull/1721#discussion_r146170251
             chunk = @stage[metadata]
@@ -435,6 +444,7 @@ module Fluent
           end
         else
           synchronize{ @stage.keys }.each do |metadata|
+            return if !force_enqueue && queue_full?
             enqueue_chunk(metadata)
           end
         end
@@ -489,6 +499,7 @@ module Fluent
 
           if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
             @metadata_list.delete(metadata)
+            @queued_num.delete(metadata)
           end
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end

data/lib/fluent/plugin/out_forward.rb

@@ -425,7 +425,7 @@ module Fluent::Plugin
     def read_ack_from_sock(sock, unpacker)
       begin
         raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
-      rescue Errno::ECONNRESET
+      rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
         raw_data = ""
       end
       info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }

data/lib/fluent/plugin/output.rb

@@ -38,7 +38,7 @@ module Fluent
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@$a-zA-Z0-9]+\}/
-      CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[\d+\])?)\}/
+      CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
       CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
 
       CHUNKING_FIELD_WARN_NUM = 4
@@ -672,7 +672,7 @@ module Fluent
         str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
           if ph == "tag"
             parts << -1
-          elsif ph =~ /^tag\[(\d+)\]$/
+          elsif ph =~ /^tag\[(-?\d+)\]$/
             parts << $1.to_i
           end
         end
@@ -708,15 +708,17 @@ module Fluent
             @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
             rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
           end
-          # ${tag}, ${tag[0]}, ${tag[1]}, ...
+          # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
           if @chunk_key_tag
             if str.include?('${tag}')
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
             if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
               hash = {}
-              metadata.tag.split('.').each_with_index do |part, i|
+              tag_parts = metadata.tag.split('.')
+              tag_parts.each_with_index do |part, i|
                 hash["${tag[#{i}]}"] = part
+                hash["${tag[#{i-tag_parts.size}]}"] = part
               end
               rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
             end
@@ -1197,7 +1199,7 @@ module Fluent
 
       def force_flush
         if @buffering
-          @buffer.enqueue_all
+          @buffer.enqueue_all(true)
           submit_flush_all
         end
       end

data/lib/fluent/time.rb

@@ -24,6 +24,7 @@ require 'fluent/config/error'
 module Fluent
   class EventTime
     TYPE = 0
+    FORMATTER = Strftime.new('%Y-%m-%d %H:%M:%S.%N %z')
 
     def initialize(sec, nsec = 0)
       @sec = sec
@@ -108,6 +109,10 @@ module Fluent
     def method_missing(name, *args, &block)
       @sec.send(name, *args, &block)
     end
+
+    def inspect
+      FORMATTER.exec(Time.at(self))
+    end
   end
 
   module TimeMixin

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.1.2'
+  VERSION = '1.1.3'
 
 end

data/test/plugin/test_output.rb

@@ -331,6 +331,24 @@ class OutputTest < Test::Unit::TestCase
       assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, c)
     end
 
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract negative index with tag' do |api|
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
+      assert @i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal ['key1','key2'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[-1]}/${tag[-2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/output/test/value1/value2/tail", @i.extract_placeholders(tmpl, c)
+    end
+
     data(:new_api => :chunk,
          :old_api => :metadata)
     test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do |api|
@@ -338,7 +356,7 @@ class OutputTest < Test::Unit::TestCase
       assert @i.chunk_key_time
       assert @i.chunk_key_tag
       assert_equal ['key1','key2'], @i.chunk_keys
-      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail"
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[-4]}/${key3}/${key4}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
       c = if api == :chunk
@@ -483,6 +501,9 @@ class OutputTest < Test::Unit::TestCase
         assert_nothing_raised do
           @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
         end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[-1]}.log")
+        end
       end
 
       test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do

data/test/plugin/test_output_as_buffered_retries.rb

@@ -412,6 +412,68 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       assert{ @i.buffer.stage.size == 1 }
       assert{ chunks.all?{|c| c.empty? } }
     end
+
+    test 'output plugin limits queued chunks via queued_chunks_limit_size' do
+      chunk_key = 'tag'
+      hash = {
+        'flush_interval' => 1,
+        'flush_thread_burst_interval' => 0.1,
+        'retry_randomize' => false,
+        'retry_max_times' => 7,
+        'queued_chunks_limit_size' => 2,
+      }
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
+      @i.register(:prefer_buffered_processing) { true }
+      @i.register(:format) { |tag,time,record| [tag,time.to_i,record].to_json + "\n" }
+      @i.register(:write) { |chunk| raise "yay, your #write must fail" }
+      @i.start
+      @i.after_start
+
+      @i.interrupt_flushes
+
+      now = Time.parse('2016-04-13 18:33:30 -0700')
+      Timecop.freeze(now)
+
+      @i.emit_events("test.tag.1", dummy_event_stream())
+
+      now = Time.parse('2016-04-13 18:33:31 -0700')
+      Timecop.freeze(now)
+
+      @i.emit_events("test.tag.2", dummy_event_stream())
+
+      @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
+      waiting(4) { Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
+
+      assert { @i.buffer.queue.size > 0 }
+      assert { @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
+
+      assert { @i.write_count > 0 }
+      assert { @i.num_errors > 0 }
+
+      prev_write_count = @i.write_count
+      prev_num_errors = @i.num_errors
+
+      chunks = @i.buffer.queue.dup
+
+      20.times do |i| # large times enough
+        now = @i.next_flush_time
+
+        Timecop.freeze(now)
+        @i.enqueue_thread_wait
+        @i.flush_thread_wakeup
+        waiting(4) { Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }
+
+        @i.emit_events("test.tag.1", dummy_event_stream())
+        assert { @i.buffer.queue.size <= 2 }
+        assert { @i.buffer.stage.size == 1 } # all new data is stored into staged chunk
+
+        break if @i.buffer.queue.size == 0
+
+        prev_write_count = @i.write_count
+        prev_num_errors = @i.num_errors
+      end
+    end
   end
 
   sub_test_case 'bufferd output for retries with periodical retry' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-19 00:00:00.000000000 Z
+date: 2018-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack