fluentd 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee967542a2be7009af606bfb3b78f0a8a9481d4649943e36cb6274629b235fde
-  data.tar.gz: ceba4cc28ba327a073df7520266570f5d59b43fb1bdb7952aa4c24297d4ce3ce
+  metadata.gz: db2e0fe87d5466ddca212327e3e0ad56bd7c6055daffd05bc351c5544153908d
+  data.tar.gz: b6b92ecd3814b496dfa7522850876bffeb6f2b59278159462b3de2e91a613170
 SHA512:
-  metadata.gz: 0ac3ac045228b9d883c00248e44202f97975ea4ad320b868823e8fc15d0f68f1007012494777760fe3aff695290321776b54b25ed494ae00e190123e928f5f9e
-  data.tar.gz: f9360951581699824d0eef3646c34dac7b555555e1f578cb756605ca9fef8c2fe66c3443195e45d1ff57fae08e9d49c4c90d404b48c91d53494ff31181ba4feb
+  metadata.gz: c359fac49d0273d40d2b8452eacc3c85d899264d44f9eb57e11bcdf73373b146f065dc0d2cee87b8542a4da3087a917d344809697e1e8065165066530eda26b5
+  data.tar.gz: a080e5d6948fe33bbc02ebfba49fb851c1facd9cf6a30a38a5f3762cb8e629e93950b3cf9d24780215655c12a7ca69a7cdaf546bed31bdab8319e8dcbdd9e405

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.2
 
+## Release v1.2.2 - 2018/06/12
+
+### Enhancements
+
+* filter_parser: Add remove_key_name_field parameter
+  https://github.com/fluent/fluentd/pull/2012
+* fluent-plugin-config-format: Dump config_argument
+  https://github.com/fluent/fluentd/pull/2003
+
+### Bug fixes
+
+* in_tail: Change pos file entry handling to avoid read conflict for other plugins
+  https://github.com/fluent/fluentd/pull/1963
+* buffer: Wait for all chunks being purged before deleting @queued_num items
+  https://github.com/fluent/fluentd/pull/2016
+
 ## Release v1.2.1 - 2018/05/23
 
 ### Enhancements

data/README.md

@@ -5,7 +5,7 @@ Fluentd: Open-Source Log Collector
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
-[Fluentd](https://fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
+[Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
 <img src="https://docs.fluentd.org/images/fluentd-architecture.png" width="500px"/>

data/lib/fluent/config/configure_proxy.rb

@@ -370,6 +370,12 @@ module Fluent
 
       def dump_config_definition
         dumped_config = {}
+        if @argument
+          argument_name, _block, options = @argument
+          options[:required] = !@defaults.key?(argument_name)
+          options[:argument] = true
+          dumped_config[argument_name] = options
+        end
         @params.each do |name, config|
           dumped_config[name] = config[1]
           dumped_config[name][:required] = !@defaults.key?(name)
@@ -378,7 +384,7 @@ module Fluent
         end
         # Overwrite by config_set_default
         @defaults.each do |name, value|
-          if @params.key?(name)
+          if @params.key?(name) || @argument.first == name
             dumped_config[name][:default] = value
           else
             dumped_config[name] = { default: value }

data/lib/fluent/plugin/buffer.rb

@@ -151,6 +151,7 @@ module Fluent
         @queue = []    #=> Array (chunks)           : already flushed (not written)
         @dequeued = {} #=> Hash (unique_id -> chunk): already written (not purged)
         @queued_num = {} # metadata => int (number of queued chunks)
+        @dequeued_num = {} # metadata => int (number of dequeued chunks)
 
         @stage_size = @queue_size = 0
         @metadata_list = [] # keys of @stage
@@ -462,6 +463,8 @@ module Fluent
 
           @dequeued[chunk.unique_id] = chunk
           @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero
+          @dequeued_num[chunk.metadata] ||= 0
+          @dequeued_num[chunk.metadata] += 1
           log.trace "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
           chunk
         end
@@ -476,6 +479,7 @@ module Fluent
           @queue.unshift(chunk)
           log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
           @queued_num[chunk.metadata] += 1 # BUG if nil
+          @dequeued_num[chunk.metadata] -= 1
         end
         true
       end
@@ -497,9 +501,11 @@ module Fluent
             log.error_backtrace
           end
 
-          if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
+          @dequeued_num[chunk.metadata] -= 1
+          if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) && @dequeued_num[metadata].zero?
             @metadata_list.delete(metadata)
             @queued_num.delete(metadata)
+            @dequeued_num.delete(metadata)
           end
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end

data/lib/fluent/plugin/filter_parser.rb

@@ -32,6 +32,8 @@ module Fluent::Plugin
     config_param :reserve_data, :bool, default: false
     desc 'Keep original event time in parsed result.'
     config_param :reserve_time, :bool, default: false
+    desc 'Remove "key_name" field from the record when parsing is succeeded'
+    config_param :remove_key_name_field, :bool, default: false
     desc 'Store parsed values with specified key name prefix.'
     config_param :inject_key_prefix, :string, default: nil
     desc 'If true, invalid string is replaced with safe characters and re-parse it.'
@@ -75,6 +77,7 @@ module Fluent::Plugin
                 else
                   t.nil? ? time : t
                 end
+            @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
             return t, r
           else

data/lib/fluent/plugin/in_tail.rb

@@ -862,7 +862,7 @@ module Fluent::Plugin
         @file.write "0000000000000000\t0000000000000000\n"
         @last_pos = @file.pos
 
-        @map[path] = FilePositionEntry.new(@file, seek)
+        @map[path] = FilePositionEntry.new(@file, seek, 0, 0)
       end
 
       def self.parse(file)
@@ -874,8 +874,10 @@ module Fluent::Plugin
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
           next unless m
           path = m[1]
+          pos = m[2].to_i(16)
+          ino = m[3].to_i(16)
           seek = file.pos - line.bytesize + path.bytesize + 1
-          map[path] = FilePositionEntry.new(file, seek)
+          map[path] = FilePositionEntry.new(file, seek, pos, ino)
         }
         new(file, map, file.pos)
       end
@@ -908,16 +910,18 @@ module Fluent::Plugin
       LN_OFFSET = 33
       SIZE = 34
 
-      def initialize(file, seek)
+      def initialize(file, seek, pos, inode)
         @file = file
         @seek = seek
-        @pos = nil
+        @pos = pos 
+        @inode = inode
       end
 
       def update(ino, pos)
         @file.pos = @seek
         @file.write "%016x\t%016x" % [pos, ino]
         @pos = pos
+        @inode = ino
       end
 
       def update_pos(pos)
@@ -927,17 +931,11 @@ module Fluent::Plugin
       end
 
       def read_inode
-        @file.pos = @seek + INO_OFFSET
-        raw = @file.read(16)
-        raw ? raw.to_i(16) : 0
+        @inode
       end
 
       def read_pos
-        @pos ||= begin
-          @file.pos = @seek
-          raw = @file.read(16)
-          raw ? raw.to_i(16) : 0
-        end
+        @pos
       end
     end
 

data/lib/fluent/supervisor.rb

@@ -190,7 +190,7 @@ module Fluent
       if config[:worker_pid]
         config[:worker_pid].each_value do |pid|
           Process.kill(:USR1, pid)
-          # don't rescue Erro::ESRSH here (invalid status)
+          # don't rescue Errno::ESRCH here (invalid status)
         end
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.2.1'
+  VERSION = '1.2.2'
 
 end

data/test/command/test_plugin_config_formatter.rb

@@ -132,6 +132,7 @@ helpers: inject,compat_parameters
 time_as_integer: bool: (false)
 slow_flush_log_threshold: float: (20.0)
 <buffer>: optional, single
+ chunk_keys: array: ([])
  @type: string: ("memory")
  timekey: time: (nil)
  timekey_wait: time: (600)
@@ -154,7 +155,6 @@ slow_flush_log_threshold: float: (20.0)
  retry_exponential_backoff_base: float: (2)
  retry_max_interval: time: (nil)
  retry_randomize: bool: (true)
- chunk_keys: : ([])
 <secondary>: optional, single
  @type: string: (nil)
  <buffer>: optional, single

data/test/config/test_configure_proxy.rb

@@ -435,6 +435,43 @@ module Fluent::Config
           assert_equal(expected, @proxy.dump_config_definition)
         end
 
+        test 'plain proxy w/ argument' do
+          @proxy.instance_eval do
+            config_argument(:argname, :string)
+            config_param(:name, :string, default: "name1")
+          end
+          expected = {
+            argname: { type: :string, required: true, argument: true },
+            name: { type: :string, default: "name1", required: false }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
+        end
+
+        test 'plain proxy w/ argument default value' do
+          @proxy.instance_eval do
+            config_argument(:argname, :string, default: "value")
+            config_param(:name, :string, default: "name1")
+          end
+          expected = {
+            argname: { type: :string, default: "value", required: false, argument: true },
+            name: { type: :string, default: "name1", required: false }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
+        end
+
+        test 'plain proxy w/ argument overwriting default value' do
+          @proxy.instance_eval do
+            config_argument(:argname, :string)
+            config_param(:name, :string, default: "name1")
+            config_set_default(:argname, "value1")
+          end
+          expected = {
+            argname: { type: :string, default: "value1", required: false, argument: true },
+            name: { type: :string, default: "name1", required: false }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
+        end
+
         test 'single sub proxy' do
           @proxy.config_section(:sub) do
             config_param(:name, :string, default: "name1")

data/test/plugin/test_filter_parser.rb

@@ -209,10 +209,13 @@ class ParserFilterTest < Test::Unit::TestCase
 
   end
 
-  def test_filter_with_reserved_data
+  data(:keep_key_name => false,
+       :remove_key_name => true)
+  def test_filter_with_reserved_data(remove_key_name)
     d1 = create_driver(%[
       key_name data
       reserve_data yes
+      remove_key_name_field #{remove_key_name}
       <parse>
         @type regexp
         expression /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
@@ -230,6 +233,7 @@ class ParserFilterTest < Test::Unit::TestCase
     d2 = create_driver(%[
       key_name data
       reserve_data yes
+      remove_key_name_field #{remove_key_name}
       <parse>
         @type json
       </parse>
@@ -244,7 +248,11 @@ class ParserFilterTest < Test::Unit::TestCase
 
     first = filtered[0]
     assert_equal_event_time time, first[0]
-    assert_equal '{"xxx":"first","yyy":"second"}', first[1]['data']
+    if remove_key_name
+      assert_not_include first[1], 'data'
+    else
+      assert_equal '{"xxx":"first","yyy":"second"}', first[1]['data']
+    end
     assert_equal 'first', first[1]['xxx']
     assert_equal 'second', first[1]['yyy']
 

data/test/plugin_helper/test_child_process.rb

@@ -510,7 +510,7 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can specify subprocess name' do
       io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
-      process_naming_enabled = (open("|ps"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
+      process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
       Process.kill(:TERM, io.pid) rescue nil
       io.close rescue nil
 
@@ -527,15 +527,15 @@ class ChildProcessTest < Test::Unit::TestCase
           m.lock
           ran = true
           pids << @d.child_process_id
-          proc_lines += open("|ps"){|_io| _io.readlines }
+          proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
           m.unlock
           readio.read
         end
         sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
         m.lock
         pid = pids.first
-        # 51358 ttys001    0:00.00 sleeper -e sleep 10
-        assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[3] == "sleeeeeeeeeper" }
+        # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
+        assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
         @d.stop; @d.shutdown; @d.close; @d.terminate
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-23 00:00:00.000000000 Z
+date: 2018-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack