fluentd222 1.16.2-x86_64-linux

data/test/test_config.rb

@@ -0,0 +1,369 @@
+require_relative 'helper'
+require 'fluent/config'
+require 'fluent/config/parser'
+require 'fluent/supervisor'
+require 'fluent/load'
+require 'fileutils'
+
+class ConfigTest < Test::Unit::TestCase
+  include Fluent
+
+  TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
+
+  def read_config(path, use_yaml: false)
+    path = File.expand_path(path)
+    if use_yaml
+      context = Kernel.binding
+
+      s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+      Fluent::Config::YamlParser::Parser.new(s).build.to_element
+    else
+      File.open(path) { |io|
+        Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+      }
+    end
+  end
+
+  def prepare_config
+    write_config "#{TMP_DIR}/config_test_1.conf", %[
+      k1 root_config
+      include dir/config_test_2.conf  #
+      include #{TMP_DIR}/config_test_4.conf
+      include file://#{TMP_DIR}/config_test_5.conf
+      <include config.d/*.conf />
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+      k2 relative_path_include
+      include ../config_test_3.conf
+    ]
+    write_config "#{TMP_DIR}/config_test_3.conf", %[
+      k3 relative_include_in_included_file
+    ]
+    write_config "#{TMP_DIR}/config_test_4.conf", %[
+      k4 absolute_path_include
+    ]
+    write_config "#{TMP_DIR}/config_test_5.conf", %[
+      k5 uri_include
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+      k6 wildcard_include_1
+      <elem1 name>
+        include normal_parameter
+      </elem1>
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+      k7 wildcard_include_2
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+      <elem2 name>
+        <include ../dir/config_test_9.conf />
+      </elem2>
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+      k9 embedded
+      <elem3 name>
+        nested nested_value
+        include hoge
+      </elem3>
+    ]
+    write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+      k8 wildcard_include_3
+      <elem4 name>
+        include normal_parameter
+      </elem4>
+    ]
+
+  end
+
+  def test_include
+    prepare_config
+    c = read_config("#{TMP_DIR}/config_test_1.conf")
+    assert_equal 'root_config', c['k1']
+    assert_equal 'relative_path_include', c['k2']
+    assert_equal 'relative_include_in_included_file', c['k3']
+    assert_equal 'absolute_path_include', c['k4']
+    assert_equal 'uri_include', c['k5']
+    assert_equal 'wildcard_include_1', c['k6']
+    assert_equal 'wildcard_include_2', c['k7']
+    assert_equal 'wildcard_include_3', c['k8']
+    assert_equal [
+      'k1',
+      'k2',
+      'k3',
+      'k4',
+      'k5',
+      'k8', # Because of the file name this comes first.
+      'k6',
+      'k7',
+    ], c.keys
+
+    elem1 = c.elements.find { |e| e.name == 'elem1' }
+    assert_not_nil elem1
+    assert_equal 'name', elem1.arg
+    assert_equal 'normal_parameter', elem1['include']
+
+    elem2 = c.elements.find { |e| e.name == 'elem2' }
+    assert_not_nil elem2
+    assert_equal 'name', elem2.arg
+    assert_equal 'embedded', elem2['k9']
+    assert !elem2.has_key?('include')
+
+    elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+    assert_not_nil elem3
+    assert_equal 'nested_value', elem3['nested']
+    assert_equal 'hoge', elem3['include']
+  end
+
+  def test_check_not_fetchd
+    write_config "#{TMP_DIR}/config_test_not_fetched.conf", %[
+      <match dummy>
+       type          rewrite
+       add_prefix    filtered
+       <rule>
+         key     path
+         pattern ^[A-Z]+
+         replace
+       </rule>
+     </match>
+    ]
+    root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.conf")
+    match_conf = root_conf.elements.first
+    rule_conf  = match_conf.elements.first
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[key pattern replace], not_fetched
+
+    # accessing should delete
+    match_conf['type']
+    rule_conf['key']
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[pattern replace], not_fetched
+
+    # repeatedly accessing should not grow memory usage
+    before_size = match_conf.unused.size
+    10.times { match_conf['type'] }
+    assert_equal before_size, match_conf.unused.size
+  end
+
+  sub_test_case "yaml config" do
+    def test_included
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - source:
+            $type: dummy
+            tag: tag.dummy
+        - source:
+            $type: tcp
+            tag: tag.tcp
+            parse:
+              $arg:
+                - why.parse.section.doesnot.have.arg
+                - huh
+              $type: none
+        - match:
+            $tag: tag.*
+            $type: stdout
+            buffer:
+              $type: memory
+              flush_interval: 1s
+        - !include fluent-included.yaml
+      EOS
+      write_config "#{TMP_DIR}/fluent-included.yaml", <<-EOS
+      - label:
+          $name: '@FLUENT_LOG'
+          config:
+            - match:
+                $type: "null"
+                $tag: "**"
+                buffer:
+                  $type: memory
+                  flush_mode: interval
+                  flush_interval: 1s
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      dummy_source_conf = root_conf.elements.first
+      tcp_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+      label_conf = root_conf.elements[3]
+      fluent_log_conf = label_conf.elements.first
+      fluent_log_buffer_conf = fluent_log_conf.elements.first
+
+      assert_equal(
+        [
+          'dummy',
+          'tag.dummy',
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'stdout',
+          'tag.*',
+          'null',
+          '**',
+          '@FLUENT_LOG',
+          'memory',
+          'interval',
+          '1s',
+        ],
+        [
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          match_conf['@type'],
+          match_conf.arg,
+          fluent_log_conf['@type'],
+          fluent_log_conf.arg,
+          label_conf.arg,
+          fluent_log_buffer_conf['@type'],
+          fluent_log_buffer_conf['flush_mode'],
+          fluent_log_buffer_conf['flush_interval'],
+        ])
+    end
+
+    def test_included_glob
+      write_config "#{TMP_DIR}/config.yaml", <<-EOS
+      config:
+        - !include "include/*.yaml"
+      EOS
+      write_config "#{TMP_DIR}/include/02_source2.yaml", <<-EOS
+      - source:
+          $type: dummy
+          tag: tag.dummy
+      EOS
+      write_config "#{TMP_DIR}/include/01_source1.yaml", <<-EOS
+      - source:
+          $type: tcp
+          tag: tag.tcp
+          parse:
+            $arg:
+              - why.parse.section.doesnot.have.arg
+              - huh
+            $type: none
+      EOS
+      write_config "#{TMP_DIR}/include/03_match1.yaml", <<-EOS
+      - match:
+          $tag: tag.*
+          $type: stdout
+          buffer:
+            $type: memory
+            flush_interval: 1s
+      EOS
+      root_conf = read_config("#{TMP_DIR}/config.yaml", use_yaml: true)
+      tcp_source_conf = root_conf.elements.first
+      dummy_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+
+      assert_equal(
+        [
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'dummy',
+          'tag.dummy',
+          'stdout',
+          'tag.*',
+        ],
+        [
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          match_conf['@type'],
+          match_conf.arg,
+        ])
+    end
+
+    def test_check_not_fetchd
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - match:
+            $arg: dummy
+            $type: rewrite
+            add_prefix:    filtered
+            rule:
+              key:     path
+              pattern: "^[A-Z]+"
+              replace: true
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      match_conf = root_conf.elements.first
+      rule_conf  = match_conf.elements.first
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[key pattern replace], not_fetched
+
+      # accessing should delete
+      match_conf['type']
+      rule_conf['key']
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[pattern replace], not_fetched
+
+      # repeatedly accessing should not grow memory usage
+      before_size = match_conf.unused.size
+      10.times { match_conf['type'] }
+      assert_equal before_size, match_conf.unused.size
+    end
+  end
+
+  def write_config(path, data, encoding: 'utf-8')
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
+  end
+
+  sub_test_case '.build' do
+    test 'read config' do
+      write_config("#{TMP_DIR}/build/config_build.conf", 'key value')
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build.conf")
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with encoding' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "#てすと\nkey value", encoding: 'shift_jis')
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", encoding: 'shift_jis')
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with additional_config' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "key value")
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", additional_config: 'key2 value2')
+      assert_equal('value', c['key'])
+      assert_equal('value2', c['key2'])
+    end
+  end
+end

data/test/test_configdsl.rb

@@ -0,0 +1,148 @@
+require_relative 'helper'
+require 'fluent/config/dsl'
+require 'fluent/test'
+
+class ConfigDSLTest < Test::Unit::TestCase
+  # TEST_CONFIG1 = %[
+  #   <source>
+  #     type forward
+  #     port 24224
+  #   </source>
+  #   <match test.**>
+  #     type forward
+  #     flush_interval 1s
+  #     <server>
+  #       host host0.example.com
+  #       port 24224
+  #     </server>
+  #     <server>
+  #       host host1.example.com
+  #       port 24224
+  #     </server>
+  #   </match>
+  # ]
+  TEST_DSL_CONFIG1 = %q[
+source {
+  type "forward"
+  port 24224
+}
+match('test.**') {
+  type "forward"
+  flush_interval "1s"
+  (0..1).each do |i|
+    server {
+      host "host#{i}.example.com"
+      port 24224
+    }
+  end
+}
+]
+
+  TEST_DSL_CONFIG2 = %q[
+v = [0, 1, 2]
+]
+
+  TEST_DSL_CONFIG3 = %q[
+match
+]
+
+  TEST_DSL_CONFIG4 = %q[
+match('aa', 'bb'){
+  type :null
+}
+]
+
+  TEST_DSL_CONFIG5 = %q[
+match('aa')
+]
+
+  def test_parse
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG1)
+
+    assert_equal 0, root.keys.size
+    assert_equal 2, root.elements.size
+
+    e0 = root.elements[0]
+    assert_equal 'source', e0.name
+    assert_equal '', e0.arg
+    assert_equal 'forward', e0['@type']
+    assert_equal '24224', e0['port']
+
+    e1 = root.elements[1]
+    assert_equal 'match', e1.name
+    assert_equal 'test.**', e1.arg
+    assert_equal 'forward', e1['@type']
+    assert_equal '1s', e1['flush_interval']
+    assert_equal 2, e1.elements.size
+    e1s0 = e1.elements[0]
+    assert_equal 'server', e1s0.name
+    assert_equal 'host0.example.com', e1s0['host']
+    assert_equal '24224', e1s0['port']
+    e1s1 = e1.elements[1]
+    assert_equal 'server', e1s1.name
+    assert_equal 'host1.example.com', e1s1['host']
+    assert_equal '24224', e1s1['port']
+  end
+
+  def test_parse2
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG2)
+
+    assert_equal 0, root.keys.size
+    assert_equal 0, root.elements.size
+  end
+
+  def test_config_error
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG3)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG4)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)
+    }
+  end
+
+  def test_with_ruby_keyword
+    uname_string = `uname -a`
+    root1 = Fluent::Config::DSL::Parser.parse(<<DSL)
+uname_str = ruby.open('|uname -a'){|out| out.read}
+source {
+  uname uname_str
+}
+DSL
+    source1 = root1.elements.first
+    assert_equal 'source', source1.name
+    assert_equal 1, source1.keys.size
+    assert_equal uname_string, source1['uname']
+
+    root2 = Fluent::Config::DSL::Parser.parse(<<DSL)
+ruby_version = ruby {
+  require 'erb'
+  ERB.new('<%= RUBY_VERSION %> from erb').result
+}
+source {
+  version ruby_version
+}
+DSL
+    source2 = root2.elements.first
+    assert_equal 'source', source2.name
+    assert_equal 1, source2.keys.size
+    assert_equal "#{RUBY_VERSION} from erb", source2['version']
+
+    # Parser#parse raises NoMethodError when configuration dsl elements are written in ruby block
+    conf3 = <<DSL
+ruby {
+  source {
+    type "tail"
+  }
+}
+source {
+  uname uname_str
+}
+DSL
+    assert_raise (NoMethodError) { Fluent::Config::DSL::Parser.parse(conf3) }
+  end
+end

data/test/test_daemonizer.rb

@@ -0,0 +1,91 @@
+require_relative 'helper'
+require 'fluent/daemonizer'
+
+class DaemonizerTest < ::Test::Unit::TestCase
+  TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
+
+  setup do
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  teardown do
+    FileUtils.rm_rf(TMP_DIR) rescue nil
+  end
+
+  test 'makes pid file' do
+    pid_path = File.join(TMP_DIR, 'file.pid')
+
+    mock(Process).daemon(anything, anything).once
+    r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+    assert_equal 'ret', r
+    assert File.exist?(pid_path)
+    assert Process.pid.to_s, File.read(pid_path).to_s
+  end
+
+  test 'in platforms which do not support fork' do
+    pid_path = File.join(TMP_DIR, 'file.pid')
+
+    mock(Process).daemon(anything, anything) { raise NotImplementedError }
+    args = ['-c', 'test.conf']
+    mock(Process).spawn(anything, *args) { Process.pid }
+
+    Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
+    assert File.exist?(pid_path)
+    assert Process.pid.to_s, File.read(pid_path).to_s
+  end
+
+  sub_test_case 'when pid file already exists' do
+    test 'raise an error when process is running' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      pid_path = File.join(TMP_DIR, 'file.pid')
+      File.write(pid_path, '1')
+
+      mock(Process).daemon(anything, anything).never
+      mock(Process).kill(0, 1).once
+
+      assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
+        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when file is not redable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
+
+      File.write(not_readable_path, '1')
+      FileUtils.chmod(0333, not_readable_path)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
+        Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when file is not writable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
+
+      File.write(not_writable_path, '1')
+      FileUtils.chmod(0555, not_writable_path)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
+        Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when directory is not writable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_writable_dir = File.join(TMP_DIR, 'not_writable')
+      pid_path = File.join(not_writable_dir, 'file.pid')
+
+      FileUtils.mkdir_p(not_writable_dir)
+      FileUtils.chmod(0555, not_writable_dir)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
+        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+      end
+    end
+  end
+end