fluentd 1.8.0 → 1.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e87e035a4839980ff8d1a31999d731790802d1e92f1c45fe1694622d34ea59c9
-  data.tar.gz: 787e06b6b6b74ee3f93e3e066627f11e02af7d10027f2dea913d22d14b1eab7f
+  metadata.gz: 7df1551f31fc37657b4a757f55ca11bfa66c9a925662512d36ac87309c07bae6
+  data.tar.gz: 3aec1319bee2aa7a07f5a2da46925880cd701a2f004eaff8616ee42077c3f437
 SHA512:
-  metadata.gz: a56f21ab62ce38de8eb55d4f5f87340b08c603c46a64553e55475a22df5af40caf75d0768b0d87df99d7581e6a47f5344e8ffa590527d7456ee6334ec5a62d16
-  data.tar.gz: 34c24131c84b70c74201781f1ff87c3e0d5081b56f92c81144cb52dfe3ba89b2c5c2202be0d7e990ccab87945ae0333eb8a8f8a37ea3f02eb3528bcf3d2d3ff8
+  metadata.gz: 76490ade18c60a06ead5f3626d94eccff81c356278841832b4007d232c31db72a1a89e8f4ef4ff12005ee3425ee75c5484bdd2b38ce347d21f22cd53331d6e86
+  data.tar.gz: c273fe0a21377a313f5a29fbf68c15731bc8dc4fe3cb6330b6b9cf37a8921a716265ee1459d2ef322bd886e3db88411dc88efac7cc50dd498b7f797c404bda22

data/.travis.yml

@@ -13,13 +13,17 @@ matrix:
       os: linux
     - rvm: 2.2.10
       os: linux-ppc64le
-    - rvm: 2.4.6
+    - rvm: 2.4.9
       os: linux
-    - rvm: 2.4.6
+    - rvm: 2.4.9
       os: linux-ppc64le
-    - rvm: 2.5.5
+    - rvm: 2.5.7
+      os: linux
+    - rvm: 2.5.7
       os: linux
-    - rvm: 2.6.3
+      arch: s390x
+      dist: xenial
+    - rvm: 2.6.5
       os: linux
     - rvm: ruby-head
       os: linux
@@ -28,12 +32,6 @@ matrix:
     - rvm: 2.1.10
       os: osx
       osx_image: xcode8.3 # OSX 10.12
-    # - rvm: 2.2.6
-    #   os: osx
-    #   osx_image: xcode8.2 # OSX 10.12
-    # - rvm: 2.3.3
-    #   os: osx
-    #   osx_image: xcode8.2 # OSX 10.12
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3 # OSX 10.12
@@ -41,15 +39,13 @@ matrix:
       os: osx
       osx_image: xcode8.3 # OSX 10.12
   allow_failures:
-    - rvm: 2.3.8
-      os: linux
     - rvm: 2.1.10
       os: osx
       osx_image: xcode8.3
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
-    - rvm: 2.5.5
+    - rvm: 2.5.7
       os: linux
       arch: s390x
       dist: xenial

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.8
 
+## Release v1.8.1 - 2019/12/26
+
+### Enhancement
+
+* in_tail: Add `path_timezone` parameter to format `path` with the specified timezone
+  https://github.com/fluent/fluentd/pull/2719
+* out_copy: Add `copy_mode` parameter. `deep_copy` parameter is now deprecated.
+  https://github.com/fluent/fluentd/pull/2747
+* supervisor: Add deprecated log for `inline_config`
+  https://github.com/fluent/fluentd/pull/2746
+
+### Bug fixes
+
+* parser_ltsv: Prevent garbage result by checking `label_delimiter`
+  https://github.com/fluent/fluentd/pull/2748
+
 ## Release v1.8.0 - 2019/12/11
 
 ### New feature

data/lib/fluent/plugin/in_tail.rb

@@ -96,6 +96,8 @@ module Fluent::Plugin
     config_param :skip_refresh_on_startup, :bool, default: false
     desc 'Ignore repeated permission error logs'
     config_param :ignore_repeated_permission_error, :bool, default: false
+    desc 'Format path with the specified timezone'
+    config_param :path_timezone, :string, default: nil
 
     config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
       config_argument :usage, :string, default: 'in_tail_parser'
@@ -129,6 +131,11 @@ module Fluent::Plugin
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
+      if @path_timezone
+        Fluent::Timezone.validate!(@path_timezone)
+        @path_formatters = @paths.map{|path| [path, Fluent::Timezone.formatter(@path_timezone, path)]}.to_h
+        @exclude_path_formatters = @exclude_path.map{|path| [path, Fluent::Timezone.formatter(@path_timezone, path)]}.to_h
+      end
 
       # TODO: Use plugin_root_dir and storage plugin to store positions if available
       if @pos_file
@@ -225,17 +232,20 @@ module Fluent::Plugin
     end
 
     def expand_paths
-      date = Time.now
+      date = Fluent::EventTime.now
       paths = []
-
       @paths.each { |path|
-        path = date.strftime(path)
+        path = if @path_timezone
+                 @path_formatters[path].call(date)
+               else
+                 date.to_time.strftime(path)
+               end
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
               if File.readable?(p) && is_file
-                if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
+                if @limit_recently_modified && File.mtime(p) < (date.to_time - @limit_recently_modified)
                   false
                 else
                   true
@@ -259,7 +269,14 @@ module Fluent::Plugin
           paths << path
         end
       }
-      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
+      excluded = @exclude_path.map { |path|
+        path = if @path_timezone
+                 @exclude_path_formatters[path].call(date)
+               else
+                 date.to_time.strftime(path)
+               end
+        path.include?('*') ? Dir.glob(path) : path
+      }.flatten.uniq
       paths - excluded
     end
 

data/lib/fluent/plugin/out_copy.rb

@@ -23,7 +23,9 @@ module Fluent::Plugin
     Fluent::Plugin.register_output('copy', self)
 
     desc 'If true, pass different record to each `store` plugin.'
-    config_param :deep_copy, :bool, default: false
+    config_param :deep_copy, :bool, default: false, deprecated: "use 'copy_mode' parameter instead"
+    desc 'Pass different record to each `store` plugin by specified method'
+    config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
 
     attr_reader :ignore_errors
 
@@ -35,6 +37,7 @@ module Fluent::Plugin
     def configure(conf)
       super
 
+      @copy_proc = gen_copy_proc
       @stores.each { |store|
         @ignore_errors << (store.arg == 'ignore_error')
       }
@@ -55,7 +58,7 @@ module Fluent::Plugin
 
       outputs.each_with_index do |output, i|
         begin
-          output.emit_events(tag, @deep_copy ? es.dup : es)
+          output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
         rescue => e
           if @ignore_errors[i]
             log.error "ignore emit error", error: e
@@ -65,5 +68,40 @@ module Fluent::Plugin
         end
       end
     end
+
+    private
+
+    def gen_copy_proc
+      @copy_mode = :shallow if @deep_copy
+
+      case @copy_mode
+      when :no_copy
+         nil
+      when :shallow
+        Proc.new { |es| es.dup }
+      when :deep
+        Proc.new { |es|
+          packer = Fluent::MessagePackFactory.msgpack_packer
+          times = []
+          records = []
+          es.each { |time, record|
+            times << time
+            packer.pack(record)
+          }
+          Fluent::MessagePackFactory.msgpack_unpacker.feed_each(packer.full_pack) { |record|
+            records << record
+          }
+          Fluent::MultiEventStream.new(times, records)
+        }
+      when :marshal
+        Proc.new { |es|
+          new_es = Fluent::MultiEventStream.new
+          es.each { |time, record|
+            new_es.add(time, Marshal.load(Marshal.dump(record)))
+          }
+          new_es
+        }
+      end
+    end
   end
 end

data/lib/fluent/plugin/parser_ltsv.rb

@@ -38,8 +38,10 @@ module Fluent
       def parse(text)
         r = {}
         text.split(@delimiter).each do |pair|
-          key, value = pair.split(@label_delimiter, 2)
-          r[key] = value
+          if pair.include? @label_delimiter
+            key, value = pair.split(@label_delimiter, 2)
+            r[key] = value
+          end
         end
         time, record = convert_values(parse_time(r), r)
         yield time, record

data/lib/fluent/root_agent.rb

@@ -163,7 +163,6 @@ module Fluent
     def setup_error_label(e)
       error_label = add_label(ERROR_LABEL)
       error_label.configure(e)
-      error_label.root_agent = RootAgentProxyWithoutErrorCollector.new(self)
       @error_collector = error_label.event_router
     end
 
@@ -363,34 +362,5 @@ module Fluent
         raise error
       end
     end
-
-    # <label @ERROR> element use RootAgent wrapped by # this RootAgentProxyWithoutErrorCollector.
-    # So that those elements don't send cause infinite loop.
-    class RootAgentProxyWithoutErrorCollector < SimpleDelegator
-      def initialize(root_agent)
-        super
-
-        @suppress_emit_error_log_interval = 0
-        @next_emit_error_log_time = nil
-
-        interval_time = root_agent.instance_variable_get(:@suppress_emit_error_log_interval)
-        suppress_interval(interval_time) unless interval_time.zero?
-      end
-
-      def emit_error_event(tag, time, record, error)
-        error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
-        log.warn "dump an error event in @ERROR:", error_info
-      end
-
-      def handle_emits_error(tag, es, e)
-        now = EventTime.now.to_i
-        if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
-          log.warn_backtrace
-          @next_emit_error_log_time = now + @suppress_emit_error_log_interval
-        end
-        raise e
-      end
-    end
   end
 end

data/lib/fluent/supervisor.rb

@@ -555,6 +555,7 @@ module Fluent
       end
 
       if @inline_config == '-'
+        $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
         @inline_config = STDIN.read
       end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.8.0'
+  VERSION = '1.8.1'
 
 end

data/test/plugin/test_in_tail.rb

@@ -969,6 +969,27 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
     end
 
+    def test_expand_paths_with_timezone
+      ['Asia/Taipei', '+08'].each do |tz_type|
+        taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
+        plugin = create_driver(taipei_config, false).instance
+
+        # Test exclude
+        exclude_config = taipei_config + config_element("", "", { "exclude_path" => %Q(["test/plugin/**/%Y%m%d-%H%M%S.log"]) })
+        exclude_plugin = create_driver(exclude_config, false).instance
+
+        with_timezone('utc') do
+          flexstub(Time) do |timeclass|
+            # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
+            timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
+
+            assert_equal EX_PATHS, plugin.expand_paths.sort
+            assert_equal EX_PATHS - [EX_PATHS.first], exclude_plugin.expand_paths.sort
+          end
+        end
+      end
+    end
+
     def test_log_file_without_extension
       expected_files = [
         'test/plugin/data/log/bar',

data/test/plugin/test_out_copy.rb

@@ -50,6 +50,22 @@ class CopyOutputTest < Test::Unit::TestCase
     assert_equal "c0", outputs[0].name
     assert_equal "c1", outputs[1].name
     assert_equal "c2", outputs[2].name
+    assert_false d.instance.deep_copy
+    assert_equal :no_copy, d.instance.copy_mode
+  end
+
+  def test_configure_with_deep_copy_and_use_shallow_copy_mode
+    d = create_driver(%[
+      deep_copy true
+      <store>
+        @type test
+        name c0
+      </store>
+    ])
+
+    outputs = d.instance.outputs
+    assert_true d.instance.deep_copy
+    assert_equal :shallow, d.instance.copy_mode
   end
 
   def test_feed_events
@@ -86,9 +102,9 @@ class CopyOutputTest < Test::Unit::TestCase
     }
   end
 
-  def create_event_test_driver(does_deep_copy = false)
+  def create_event_test_driver(copy_mode = 'no_copy')
     config = %[
-      deep_copy #{does_deep_copy}
+      copy_mode #{copy_mode}
       <store>
         @type test
         name output1
@@ -110,49 +126,60 @@ class CopyOutputTest < Test::Unit::TestCase
   end
 
   time = event_time("2013-05-26 06:37:22 UTC")
-  mes0 = Fluent::MultiEventStream.new
-  mes0.add(time, {"a" => 1})
-  mes0.add(time, {"b" => 1})
-  mes1 = Fluent::MultiEventStream.new
-  mes1.add(time, {"a" => 1})
-  mes1.add(time, {"b" => 1})
+  gen_multi_es = Proc.new {
+    es = Fluent::MultiEventStream.new
+    es.add(time, {"a" => 1, "nest" => {'k' => 'v'}})
+    es.add(time, {"b" => 1, "nest" => {'k' => 'v'}})
+    es
+  }
 
   data(
-    "OneEventStream without deep_copy"   => [false, Fluent::OneEventStream.new(time, {"a" => 1})],
-    "OneEventStream with deep_copy"      => [true,  Fluent::OneEventStream.new(time, {"a" => 1})],
-    "ArrayEventStream without deep_copy" => [false, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
-    "ArrayEventStream with deep_copy"    => [true,  Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
-    "MultiEventStream without deep_copy" => [false, mes0],
-    "MultiEventStream with deep_copy"    => [true,  mes1],
+    "OneEventStream without copy" => ['no_copy', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
+    "OneEventStream with shallow" => ['shallow', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
+    "OneEventStream with marshal" => ['marshal', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
+    "OneEventStream with deep"    => ['deep',    Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
+    "ArrayEventStream without copy" => ['no_copy', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
+    "ArrayEventStream with shallow" => ['shallow', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
+    "ArrayEventStream with marshal" => ['marshal', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
+    "ArrayEventStream with deep"    => ['deep',    Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
+    "MultiEventStream without copy" => ['no_copy', gen_multi_es.call],
+    "MultiEventStream with shallow" => ['shallow', gen_multi_es.call],
+    "MultiEventStream with marshal" => ['marshal', gen_multi_es.call],
+    "MultiEventStream with deep"    => ['deep',    gen_multi_es.call],
   )
-  def test_deep_copy_controls_shallow_or_deep_copied(data)
-    does_deep_copy, es = data
-
-    d = create_event_test_driver(does_deep_copy)
+  def test_copy_mode_with_event_streams(data)
+    copy_mode, es = data
 
+    d = create_event_test_driver(copy_mode)
     d.run(default_tag: 'test') do
       d.feed(es)
     end
 
     events = d.instance.outputs.map(&:events)
 
-    if does_deep_copy
+    if copy_mode != 'no_copy'
       events[0].each_with_index do |entry0, i|
         record0 = entry0.last
         record1 = events[1][i].last
 
-        assert{ record0.object_id != record1.object_id }
+        assert_not_equal record0.object_id, record1.object_id
         assert_equal "bar", record0["foo"]
         assert !record1.has_key?("foo")
+        if copy_mode == 'shallow'
+          assert_equal record0['nest'].object_id, record1['nest'].object_id
+        else
+          assert_not_equal record0['nest'].object_id, record1['nest'].object_id
+        end
       end
     else
       events[0].each_with_index do |entry0, i|
         record0 = entry0.last
         record1 = events[1][i].last
 
-        assert{ record0.object_id == record1.object_id }
+        assert_equal record0.object_id, record1.object_id
         assert_equal "bar", record0["foo"]
         assert_equal "bar", record1["foo"]
+        assert_equal record0['nest'].object_id, record1['nest'].object_id
       end
     end
   end

data/test/plugin/test_parser_labeled_tsv.rb

@@ -100,6 +100,21 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
     end
   end
 
+  def test_parse_and_reject_invalid_kv_pairs
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure(
+                     'delimiter'       => ' ',
+                     'label_delimiter' => '=',
+                     )
+    text = 'A leading portion that is not LTSV    :  foo=bar baz=derp and a trailing portion'
+
+    expected = {'foo' => 'bar', 'baz' => 'derp'}
+    parser.instance.parse(text) do |time, record|
+      assert_equal expected, record
+    end
+
+  end
+
   def test_parse_with_null_value_pattern
     parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
     parser.configure(

data/test/test_root_agent.rb

@@ -170,7 +170,6 @@ EOC
 
       error_label = ra.labels['@ERROR']
       assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first
-      assert_kind_of RootAgent::RootAgentProxyWithoutErrorCollector, error_label.root_agent
     end
   end
 
@@ -853,7 +852,6 @@ EOC
 
       error_label = ra.labels['@ERROR']
       assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first
-      assert_kind_of RootAgent::RootAgentProxyWithoutErrorCollector, error_label.root_agent
     end
 
     test 'with plugins but for another worker' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.8.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-12 00:00:00.000000000 Z
+date: 2019-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack