fluentd 1.8.1 → 1.9.0.rc1

data/test/test_output.rb

@@ -116,7 +116,7 @@ module FluentOutputTest
         end
       end
 
-      mock(d.instance.log).warn("secondary type should be same with primary one",
+      mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                                 { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
       d.configure(CONFIG + %[
         <secondary>
@@ -132,7 +132,7 @@ module FluentOutputTest
       # ObjectBufferedOutput doesn't implement `custom_filter`
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
-      mock(d.instance.log).warn("secondary type should be same with primary one",
+      mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                                 { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
       d.configure(CONFIG + %[
         <secondary>

data/test/test_static_config_analysis.rb

@@ -0,0 +1,177 @@
+require_relative 'helper'
+
+require 'fluent/config'
+require 'fluent/static_config_analysis'
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_stdout'
+require 'fluent/plugin/out_exec'
+require 'fluent/plugin/in_forward'
+require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/filter_grep'
+require 'fluent/plugin/filter_stdout'
+require 'fluent/plugin/filter_parser'
+
+class StaticConfigAnalysisTest < ::Test::Unit::TestCase
+  sub_test_case '.call' do
+    test 'returns outputs, inputs and filters' do
+      conf_data = <<-CONF
+<source>
+  @type forward
+</source>
+<filter>
+  @type grep
+</filter>
+<match>
+  @type forward
+</match>
+      CONF
+
+      c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      ret = Fluent::StaticConfigAnalysis.call(c)
+      assert_equal 1, ret.outputs.size
+      assert_kind_of Fluent::Plugin::ForwardOutput, ret.outputs[0].plugin
+      assert_equal 1, ret.inputs.size
+      assert_kind_of Fluent::Plugin::ForwardInput, ret.inputs[0].plugin
+      assert_equal 1, ret.filters.size
+      assert_kind_of Fluent::Plugin::GrepFilter, ret.filters[0].plugin
+      assert_empty ret.labels
+
+      assert_equal [Fluent::Plugin::ForwardOutput, Fluent::Plugin::ForwardInput, Fluent::Plugin::GrepFilter], ret.all_plugins.map(&:class)
+    end
+
+    test 'returns wrapped element with worker and label section' do
+      conf_data = <<-CONF
+<source>
+  @type forward
+</source>
+<filter>
+  @type grep
+</filter>
+<match>
+  @type forward
+</match>
+<worker 0>
+  <source>
+    @type dummy
+  </source>
+  <filter>
+    @type parser
+  </filter>
+  <match>
+    @type exec
+  </match>
+</worker>
+<label @test>
+  <filter>
+    @type stdout
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+      CONF
+
+      c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      ret = Fluent::StaticConfigAnalysis.call(c)
+      assert_equal [Fluent::Plugin::ExecOutput, Fluent::Plugin::StdoutOutput, Fluent::Plugin::ForwardOutput], ret.outputs.map(&:plugin).map(&:class)
+      assert_equal [Fluent::Plugin::DummyInput, Fluent::Plugin::ForwardInput], ret.inputs.map(&:plugin).map(&:class)
+      assert_equal [Fluent::Plugin::ParserFilter, Fluent::Plugin::StdoutFilter, Fluent::Plugin::GrepFilter], ret.filters.map(&:plugin).map(&:class)
+      assert_equal 1, ret.labels.size
+      assert_equal '@test', ret.labels[0].name
+    end
+
+    sub_test_case 'raises config error' do
+      data(
+        'empty' => ['', 'Missing worker id on <worker> directive'],
+        'invalid number' => ['a', 'worker id should be integer: a'],
+        'worker id is negative' => ['-1', 'worker id should be integer: -1'],
+        'min worker id is less than 0' => ['-1-1', 'worker id should be integer: -1-1'],
+        'max worker id is less than 0' => ['1--1', 'worker id -1 specified by <worker> directive is not allowed. Available worker id is between 0 and 1'],
+        'min worker id is greater than workers' => ['0-2', 'worker id 2 specified by <worker> directive is not allowed. Available worker id is between 0 and 1'],
+        'max worker is less than min worker' => ['1-0', "greater first_worker_id<1> than last_worker_id<0> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"],
+      )
+      test 'when worker number is invalid' do |v|
+        val, msg = v
+        conf_data = <<-CONF
+<worker #{val}>
+</worker>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new(msg)) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      test 'when worker number is duplicated' do
+        conf_data = <<-CONF
+<worker 0-1>
+</worker>
+<worker 0-1>
+</worker>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new("specified worker_id<0> collisions is detected on <worker> directive. Available worker id(s): []")) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      test 'duplicated label exits' do
+        conf_data = <<-CONF
+<label @dup>
+</label>
+<label @dup>
+</label>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new('Section <label @dup> appears twice')) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      test 'empty label' do
+        conf_data = <<-CONF
+<label>
+</label>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new('Missing symbol argument on <label> directive')) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      data(
+        'in filter' => 'filter',
+        'in source' => 'source',
+        'in match' => 'match',
+      )
+      test 'when @type is missing' do |name|
+        conf_data = <<-CONF
+<#{name}>
+  @type
+</#{name}>
+CONF
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new("Missing '@type' parameter on <#{name}> directive")) do
+          Fluent::StaticConfigAnalysis.call(c)
+        end
+      end
+
+      test 'when worker has worker section' do
+        conf_data = <<-CONF
+<worker 0>
+  <worker 0>
+  </worker>
+</worker>
+CONF
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new("<worker> section cannot have <worker> directive")) do
+          Fluent::StaticConfigAnalysis.call(c)
+        end
+      end
+    end
+  end
+end

data/test/test_supervisor.rb

@@ -30,81 +30,6 @@ class SupervisorTest < ::Test::Unit::TestCase
     File.open(path, "w") {|f| f.write data }
   end
 
-  def test_read_config
-    create_info_dummy_logger
-
-    tmp_dir = "#{TMP_DIR}/dir/test_read_config.conf"
-    conf_str = %[
-<source>
-  @type forward
-  @id forward_input
-</source>
-<match debug.**>
-  @type stdout
-  @id stdout_output
-</match>
-]
-    write_config tmp_dir, conf_str
-    opts = Fluent::Supervisor.default_options
-    sv = Fluent::Supervisor.new(opts)
-
-    use_v1_config = {}
-    use_v1_config['use_v1_config'] = true
-
-    sv.instance_variable_set(:@config_path, tmp_dir)
-    sv.instance_variable_set(:@use_v1_config, use_v1_config)
-
-    conf = sv.__send__(:read_config)
-    elem = conf.elements.find { |e| e.name == 'source' }
-    assert_equal "forward", elem['@type']
-    assert_equal "forward_input", elem['@id']
-
-    elem = conf.elements.find { |e| e.name == 'match' }
-    assert_equal "debug.**", elem.arg
-    assert_equal "stdout", elem['@type']
-    assert_equal "stdout_output", elem['@id']
-
-    $log.out.reset
-  end
-
-  def test_read_config_with_multibyte_string
-    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
-    conf_str = %[
-<source>
-  @type forward
-  @id forward_input
-  @label @INPUT
-</source>
-<label @INPUT>
-  <filter>
-    @type record_transformer
-    <record>
-      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
-    </record>
-  </filter>
-  <match>
-    @type stdout
-  </match>
-</label>
-]
-    FileUtils.mkdir_p(File.dirname(tmp_path))
-    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
-
-    opts = Fluent::Supervisor.default_options
-    sv = Fluent::Supervisor.new(opts)
-
-    use_v1_config = {}
-    use_v1_config['use_v1_config'] = true
-
-    sv.instance_variable_set(:@config_path, tmp_path)
-    sv.instance_variable_set(:@use_v1_config, use_v1_config)
-
-    conf = sv.__send__(:read_config)
-    label = conf.elements.detect {|e| e.name == "label" }
-    filter = label.elements.detect {|e| e.name == "filter" }
-    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
-    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
-  end
 
   def test_system_config
     opts = Fluent::Supervisor.default_options
@@ -409,6 +334,8 @@ class SupervisorTest < ::Test::Unit::TestCase
   end
 
   def test_inline_config
+    omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711'
+
     opts = Fluent::Supervisor.default_options
     opts[:inline_config] = '-'
     sv = Fluent::Supervisor.new(opts)
@@ -416,7 +343,7 @@ class SupervisorTest < ::Test::Unit::TestCase
 
     inline_config = '<match *>\n@type stdout\n</match>'
     stub(STDIN).read { inline_config }
-    stub(sv).read_config        # to skip
+    stub(Fluent::Config).build                                # to skip
     stub(sv).build_system_config { Fluent::SystemConfig.new } # to skip
 
     sv.configure
@@ -428,7 +355,7 @@ class SupervisorTest < ::Test::Unit::TestCase
     sv = Fluent::Supervisor.new(opts)
 
     c = Fluent::Config::Element.new('system', '', { 'log_level' => 'error' }, [])
-    stub(sv).read_config { config_element('ROOT', '', {}, [c]) }
+    stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) }
 
     sv.configure
     assert_equal Fluent::Log::LEVEL_ERROR, $log.level

data/test/test_variable_store.rb

@@ -0,0 +1,65 @@
+require_relative 'helper'
+require 'fluent/variable_store'
+
+class VariableStoreTest < Test::Unit::TestCase
+  def setup
+  end
+
+  def teardown
+    Fluent::VariableStore.try_to_reset do
+      # nothing
+    end
+  end
+
+  sub_test_case '#fetch_or_build' do
+    test 'fetch same object when the same key is passed' do
+      c1 = Fluent::VariableStore.fetch_or_build(:test)
+      c2 = Fluent::VariableStore.fetch_or_build(:test)
+
+      assert_equal c1, c2
+      assert_equal c1.object_id, c2.object_id
+
+      c3 = Fluent::VariableStore.fetch_or_build(:test2)
+      assert_not_equal c1.object_id, c3.object_id
+    end
+
+    test 'can be passed a default value' do
+      c1 = Fluent::VariableStore.fetch_or_build(:test, default_value: Set.new)
+      c2 = Fluent::VariableStore.fetch_or_build(:test)
+
+      assert_kind_of Set, c1
+      assert_equal c1, c2
+      assert_equal c1.object_id, c2.object_id
+    end
+  end
+
+  sub_test_case '#try_to_reset' do
+    test 'reset all values' do
+      c1 = Fluent::VariableStore.fetch_or_build(:test)
+      c1[:k1] = 1
+      assert_equal 1, c1[:k1]
+
+      Fluent::VariableStore.try_to_reset do
+        # nothing
+      end
+
+      c1 = Fluent::VariableStore.fetch_or_build(:test)
+      assert_nil c1[:k1]
+    end
+
+    test 'rollback resetting if error raised' do
+      c1 = Fluent::VariableStore.fetch_or_build(:test)
+      c1[:k1] = 1
+      assert_equal 1, c1[:k1]
+
+      assert_raise(RuntimeError.new('pass')) do
+        Fluent::VariableStore.try_to_reset do
+          raise 'pass'
+        end
+      end
+
+      c1 = Fluent::VariableStore.fetch_or_build(:test)
+      assert_equal 1, c1[:k1]
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.9.0.rc1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-26 00:00:00.000000000 Z
+date: 2020-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -172,34 +172,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: dig_rb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: flexmock
   requirement: !ruby/object:Gem::Requirement
@@ -597,6 +583,7 @@ files:
 - lib/fluent/registry.rb
 - lib/fluent/root_agent.rb
 - lib/fluent/rpc.rb
+- lib/fluent/static_config_analysis.rb
 - lib/fluent/supervisor.rb
 - lib/fluent/system_config.rb
 - lib/fluent/test.rb
@@ -623,6 +610,7 @@ files:
 - lib/fluent/time.rb
 - lib/fluent/timezone.rb
 - lib/fluent/unique_id.rb
+- lib/fluent/variable_store.rb
 - lib/fluent/version.rb
 - lib/fluent/winsvc.rb
 - templates/new_gem/Gemfile
@@ -800,6 +788,7 @@ files:
 - test/test_clock.rb
 - test/test_config.rb
 - test/test_configdsl.rb
+- test/test_engine.rb
 - test/test_event.rb
 - test/test_event_router.rb
 - test/test_event_time.rb
@@ -818,11 +807,13 @@ files:
 - test/test_plugin_id.rb
 - test/test_process.rb
 - test/test_root_agent.rb
+- test/test_static_config_analysis.rb
 - test/test_supervisor.rb
 - test/test_test_drivers.rb
 - test/test_time_formatter.rb
 - test/test_time_parser.rb
 - test/test_unique_id.rb
+- test/test_variable_store.rb
 homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
@@ -835,12 +826,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: 
@@ -1004,6 +995,7 @@ test_files:
 - test/test_clock.rb
 - test/test_config.rb
 - test/test_configdsl.rb
+- test/test_engine.rb
 - test/test_event.rb
 - test/test_event_router.rb
 - test/test_event_time.rb
@@ -1022,8 +1014,10 @@ test_files:
 - test/test_plugin_id.rb
 - test/test_process.rb
 - test/test_root_agent.rb
+- test/test_static_config_analysis.rb
 - test/test_supervisor.rb
 - test/test_test_drivers.rb
 - test/test_time_formatter.rb
 - test/test_time_parser.rb
 - test/test_unique_id.rb
+- test/test_variable_store.rb