fluentd 0.12.40 → 1.6.2

data/test/command/test_plugin_config_formatter.rb

@@ -0,0 +1,276 @@
+require_relative '../helper'
+
+require 'pathname'
+require 'fluent/command/plugin_config_formatter'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/plugin/filter'
+require 'fluent/plugin/parser'
+require 'fluent/plugin/formatter'
+
+class TestFluentPluginConfigFormatter < Test::Unit::TestCase
+  class FakeInput < ::Fluent::Plugin::Input
+    ::Fluent::Plugin.register_input("fake", self)
+
+    desc "path to something"
+    config_param :path, :string
+  end
+
+  class FakeOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output("fake", self)
+
+    desc "path to something"
+    config_param :path, :string
+
+    def process(tag, es)
+    end
+  end
+
+  class FakeFilter < ::Fluent::Plugin::Filter
+    ::Fluent::Plugin.register_filter("fake", self)
+
+    desc "path to something"
+    config_param :path, :string
+
+    def filter(tag, time, record)
+    end
+  end
+
+  class FakeParser < ::Fluent::Plugin::Parser
+    ::Fluent::Plugin.register_parser("fake", self)
+
+    desc "path to something"
+    config_param :path, :string
+
+    def parse(text)
+    end
+  end
+
+  class FakeFormatter < ::Fluent::Plugin::Formatter
+    ::Fluent::Plugin.register_formatter("fake", self)
+
+    desc "path to something"
+    config_param :path, :string
+
+    def format(tag, time, record)
+    end
+  end
+
+  class SimpleInput < ::Fluent::Plugin::Input
+    ::Fluent::Plugin.register_input("simple", self)
+    helpers :inject, :compat_parameters
+
+    desc "path to something"
+    config_param :path, :string
+  end
+
+  class ComplexOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output("complex", self)
+    helpers :inject, :compat_parameters
+
+    config_section :authentication, required: true, multi: false do
+      desc "username"
+      config_param :username, :string
+      desc "password"
+      config_param :password, :string, secret: true
+    end
+
+    config_section :parent do
+      config_section :child do
+        desc "names"
+        config_param :names, :array
+        desc "difficulty"
+        config_param :difficulty, :enum, list: [:easy, :normal, :hard], default: :normal
+      end
+    end
+
+    def process(tag, es)
+    end
+  end
+
+  sub_test_case "json" do
+    data(input: [FakeInput, "input"],
+         output: [FakeOutput, "output"],
+         filter: [FakeFilter, "filter"],
+         parser: [FakeParser, "parser"],
+         formatter: [FakeFormatter, "formatter"])
+    test "dumped config should be valid JSON" do |(klass, type)|
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=json", type, "fake"]).call
+      end
+      expected = {
+        path: {
+          desc: "path to something",
+          type: "string",
+          required: true
+        }
+      }
+      assert_equal(expected, JSON.parse(dumped_config, symbolize_names: true)[klass.name.to_sym])
+    end
+  end
+
+  sub_test_case "text" do
+    test "input simple" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=txt", "input", "simple"]).call
+      end
+      expected = <<TEXT
+helpers: inject,compat_parameters
+@log_level: string: (nil)
+path: string: (nil)
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "output complex" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=txt", "output", "complex"]).call
+      end
+      expected = <<TEXT
+helpers: inject,compat_parameters
+@log_level: string: (nil)
+time_as_integer: bool: (false)
+slow_flush_log_threshold: float: (20.0)
+<buffer>: optional, single
+ chunk_keys: array: ([])
+ @type: string: ("memory")
+ timekey: time: (nil)
+ timekey_wait: time: (600)
+ timekey_use_utc: bool: (false)
+ timekey_zone: string: ("#{Time.now.strftime('%z')}")
+ flush_at_shutdown: bool: (nil)
+ flush_mode: enum: (:default)
+ flush_interval: time: (60)
+ flush_thread_count: integer: (1)
+ flush_thread_interval: float: (1.0)
+ flush_thread_burst_interval: float: (1.0)
+ delayed_commit_timeout: time: (60)
+ overflow_action: enum: (:throw_exception)
+ retry_forever: bool: (false)
+ retry_timeout: time: (259200)
+ retry_max_times: integer: (nil)
+ retry_secondary_threshold: float: (0.8)
+ retry_type: enum: (:exponential_backoff)
+ retry_wait: time: (1)
+ retry_exponential_backoff_base: float: (2)
+ retry_max_interval: time: (nil)
+ retry_randomize: bool: (true)
+ disable_chunk_backup: bool: (false)
+<secondary>: optional, single
+ @type: string: (nil)
+ <buffer>: optional, single
+ <secondary>: optional, single
+<authentication>: required, single
+ username: string: (nil)
+ password: string: (nil)
+<parent>: optional, multiple
+ <child>: optional, multiple
+  names: array: (nil)
+  difficulty: enum: (:normal)
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+  end
+
+  sub_test_case "markdown" do
+    test "input simple" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "input", "simple"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)
+
+## TestFluentPluginConfigFormatter::SimpleInput
+
+### path (string) (required)
+
+path to something
+
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "output complex" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "output", "complex"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)
+
+## TestFluentPluginConfigFormatter::ComplexOutput
+
+
+### \\<authentication\\> section (required) (single)
+
+#### username (string) (required)
+
+username
+
+#### password (string) (required)
+
+password
+
+
+
+### \\<parent\\> section (optional) (multiple)
+
+
+#### \\<child\\> section (optional) (multiple)
+
+##### names (array) (required)
+
+names
+
+##### difficulty (enum) (optional)
+
+difficulty
+
+Available values: easy, normal, hard
+
+Default value: `normal`.
+
+
+
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+  end
+
+  sub_test_case "arguments" do
+    data do
+      hash = {}
+      ["input", "output", "filter", "parser", "formatter"].each do |type|
+        ["txt", "json", "markdown"].each do |format|
+          argv = ["--format=#{format}"]
+          [
+            ["--verbose", "--compact"],
+            ["--verbose"],
+            ["--compact"]
+          ].each do |options|
+            hash[(argv + options).join(" ")] = argv + options + [type, "fake"]
+          end
+        end
+      end
+      hash
+    end
+    test "dump txt" do |argv|
+      capture_stdout do
+        assert_nothing_raised do
+          FluentPluginConfigFormatter.new(argv).call
+        end
+      end
+    end
+  end
+end

data/test/command/test_plugin_generator.rb

@@ -0,0 +1,92 @@
+require_relative '../helper'
+
+require 'pathname'
+require 'fluent/command/plugin_generator'
+
+class TestFluentPluginGenerator < Test::Unit::TestCase
+  TEMP_DIR = "tmp/plugin_generator"
+  setup do
+    FileUtils.mkdir_p(TEMP_DIR)
+    @pwd = Dir.pwd
+    Dir.chdir(TEMP_DIR)
+  end
+
+  teardown do
+    Dir.chdir(@pwd)
+    FileUtils.rm_rf(TEMP_DIR)
+  end
+
+  sub_test_case "generate plugin" do
+    data(input: ["input", "in"],
+         output: ["output", "out"],
+         filter: ["filter", "filter"],
+         parser: ["parser", "parser"],
+         formatter: ["formatter", "formatter"])
+    test "generate plugin" do |(type, part)|
+      capture_stdout do
+        FluentPluginGenerator.new([type, "fake"]).call
+      end
+      plugin_base_dir = Pathname("fluent-plugin-fake")
+      assert { plugin_base_dir.directory? }
+      expected = [
+        "fluent-plugin-fake",
+        "fluent-plugin-fake/Gemfile",
+        "fluent-plugin-fake/LICENSE",
+        "fluent-plugin-fake/README.md",
+        "fluent-plugin-fake/Rakefile",
+        "fluent-plugin-fake/fluent-plugin-fake.gemspec",
+        "fluent-plugin-fake/lib",
+        "fluent-plugin-fake/lib/fluent",
+        "fluent-plugin-fake/lib/fluent/plugin",
+        "fluent-plugin-fake/lib/fluent/plugin/#{part}_fake.rb",
+        "fluent-plugin-fake/test",
+        "fluent-plugin-fake/test/helper.rb",
+        "fluent-plugin-fake/test/plugin",
+        "fluent-plugin-fake/test/plugin/test_#{part}_fake.rb",
+      ]
+      actual = plugin_base_dir.find.reject {|f| f.fnmatch("*/.git*") }.map(&:to_s).sort
+      assert_equal(expected, actual)
+    end
+
+    test "no license" do
+      capture_stdout do
+        FluentPluginGenerator.new(["--no-license", "filter", "fake"]).call
+      end
+      assert { !Pathname("fluent-plugin-fake/LICENSE").exist? }
+      assert { Pathname("fluent-plugin-fake/Gemfile").exist? }
+    end
+
+    test "unknown license" do
+      out = capture_stdout do
+        assert_raise(SystemExit) do
+          FluentPluginGenerator.new(["--license=unknown", "filter", "fake"]).call
+        end
+      end
+      assert { out.lines.include?("License: unknown\n") }
+    end
+  end
+
+  sub_test_case "unify plugin name" do
+    data("word" => ["fake", "fake"],
+         "underscore" => ["rewrite_tag_filter", "rewrite_tag_filter"],
+         "dash" => ["rewrite-tag-filter", "rewrite_tag_filter"])
+    test "plugin_name" do |(name, plugin_name)|
+      generator = FluentPluginGenerator.new(["filter", name])
+      capture_stdout do
+        generator.call
+      end
+      assert_equal(plugin_name, generator.__send__(:plugin_name))
+    end
+
+    data("word" => ["fake", "fluent-plugin-fake"],
+         "underscore" => ["rewrite_tag_filter", "fluent-plugin-rewrite-tag-filter"],
+         "dash" => ["rewrite-tag-filter", "fluent-plugin-rewrite-tag-filter"])
+    test "gem_name" do |(name, gem_name)|
+      generator = FluentPluginGenerator.new(["output", name])
+      capture_stdout do
+        generator.call
+      end
+      assert_equal(gem_name, generator.__send__(:gem_name))
+    end
+  end
+end

data/test/compat/test_calls_super.rb

@@ -0,0 +1,166 @@
+require_relative '../helper'
+
+# these are Fluent::Compat::* in fact
+require 'fluent/input'
+require 'fluent/output'
+require 'fluent/filter'
+
+class CompatCallsSuperTest < Test::Unit::TestCase
+  class DummyGoodInput < Fluent::Input
+    def configure(conf); super; end
+    def start; super; end
+    def before_shutdown; super; end
+    def shutdown; super; end
+  end
+  class DummyBadInput < Fluent::Input
+    def configure(conf); super; end
+    def start; end
+    def before_shutdown; end
+    def shutdown; end
+  end
+  class DummyGoodOutput < Fluent::Output
+    def configure(conf); super; end
+    def start; super; end
+    def before_shutdown; super; end
+    def shutdown; super; end
+  end
+  class DummyBadOutput < Fluent::Output
+    def configure(conf); super; end
+    def start; end
+    def before_shutdown; end
+    def shutdown; end
+  end
+  class DummyGoodFilter < Fluent::Filter
+    def configure(conf); super; end
+    def filter(tag, time, record); end
+    def start; super; end
+    def before_shutdown; super; end
+    def shutdown; super; end
+  end
+  class DummyBadFilter < Fluent::Filter
+    def configure(conf); super; end
+    def filter(tag, time, record); end
+    def start; end
+    def before_shutdown; end
+    def shutdown; end
+  end
+
+  setup do
+    Fluent::Test.setup
+  end
+
+  sub_test_case 'old API plugin which calls super properly' do
+    test 'Input#start, #before_shutdown and #shutdown calls all superclass methods properly' do
+      i = DummyGoodInput.new
+      i.configure(config_element())
+      assert i.configured?
+
+      i.start
+      assert i.started?
+
+      i.before_shutdown
+      assert i.before_shutdown?
+
+      i.shutdown
+      assert i.shutdown?
+
+      assert i.log.out.logs.empty?
+    end
+
+    test 'Output#start, #before_shutdown and #shutdown calls all superclass methods properly' do
+      i = DummyGoodOutput.new
+      i.configure(config_element())
+      assert i.configured?
+
+      i.start
+      assert i.started?
+
+      i.before_shutdown
+      assert i.before_shutdown?
+
+      i.shutdown
+      assert i.shutdown?
+
+      assert i.log.out.logs.empty?
+    end
+
+    test 'Filter#start, #before_shutdown and #shutdown calls all superclass methods properly' do
+      i = DummyGoodFilter.new
+      i.configure(config_element())
+      assert i.configured?
+
+      i.start
+      assert i.started?
+
+      i.before_shutdown
+      assert i.before_shutdown?
+
+      i.shutdown
+      assert i.shutdown?
+
+      assert i.log.out.logs.empty?
+    end
+  end
+
+  sub_test_case 'old API plugin which does not call super' do
+    test 'Input#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do
+      i = DummyBadInput.new
+      i.configure(config_element())
+      assert i.configured?
+
+      i.start
+      assert i.started?
+
+      i.before_shutdown
+      assert i.before_shutdown?
+
+      i.shutdown
+      assert i.shutdown?
+
+      logs = i.log.out.logs
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadInput") } }
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadInput") } }
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadInput") } }
+    end
+
+    test 'Output#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do
+      i = DummyBadOutput.new
+      i.configure(config_element())
+      assert i.configured?
+
+      i.start
+      assert i.started?
+
+      i.before_shutdown
+      assert i.before_shutdown?
+
+      i.shutdown
+      assert i.shutdown?
+
+      logs = i.log.out.logs
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadOutput") } }
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadOutput") } }
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadOutput") } }
+    end
+
+    test 'Filter#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do
+      i = DummyBadFilter.new
+      i.configure(config_element())
+      assert i.configured?
+
+      i.start
+      assert i.started?
+
+      i.before_shutdown
+      assert i.before_shutdown?
+
+      i.shutdown
+      assert i.shutdown?
+
+      logs = i.log.out.logs
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadFilter") } }
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadFilter") } }
+      assert{ logs.any?{|l| l.include?("[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadFilter") } }
+    end
+  end
+end

data/test/compat/test_parser.rb

@@ -0,0 +1,92 @@
+require_relative '../helper'
+require 'fluent/plugin/parser'
+
+class TextParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  class MultiEventTestParser < ::Fluent::Parser
+    include Fluent::Configurable
+
+    def parse(text)
+      2.times { |i|
+        record = {}
+        record['message'] = text
+        record['number'] = i
+        yield Fluent::Engine.now, record
+      }
+    end
+  end
+
+  Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
+
+  def test_lookup_unknown_format
+    assert_raise Fluent::ConfigError do
+      Fluent::Plugin.new_parser('unknown')
+    end
+  end
+
+  data('register_formatter' => 'known', 'register_template' => 'known_old')
+  def test_lookup_known_parser(data)
+    $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), '..', 'scripts'))
+    assert_nothing_raised Fluent::ConfigError do
+      Fluent::Plugin.new_parser(data)
+    end
+    $LOAD_PATH.shift
+  end
+
+  def test_parse_with_return
+    parser = Fluent::TextParser.new
+    parser.configure('format' => 'none')
+    _time, record = parser.parse('log message!')
+    assert_equal({'message' => 'log message!'}, record)
+  end
+
+  def test_parse_with_block
+    parser = Fluent::TextParser.new
+    parser.configure('format' => 'none')
+    parser.parse('log message!') { |time, record|
+      assert_equal({'message' => 'log message!'}, record)
+    }
+  end
+
+  def test_multi_event_parser
+    parser = Fluent::TextParser.new
+    parser.configure('format' => 'multi_event_test')
+    i = 0
+    parser.parse('log message!') { |time, record|
+      assert_equal('log message!', record['message'])
+      assert_equal(i, record['number'])
+      i += 1
+    }
+  end
+
+  def test_setting_estimate_current_event_value
+    p1 = Fluent::TextParser.new
+    assert_nil p1.estimate_current_event
+    assert_nil p1.parser
+
+    p1.configure('format' => 'none')
+    assert_equal true, p1.parser.estimate_current_event
+
+    p2 = Fluent::TextParser.new
+    assert_nil p2.estimate_current_event
+    assert_nil p2.parser
+
+    p2.estimate_current_event = false
+
+    p2.configure('format' => 'none')
+    assert_equal false, p2.parser.estimate_current_event
+  end
+
+  data(ignorecase: Regexp::IGNORECASE,
+       multiline: Regexp::MULTILINE,
+       both: Regexp::IGNORECASE & Regexp::MULTILINE)
+  def test_regexp_parser_config(options)
+    source = "(?<test>.*)"
+    parser = Fluent::TextParser::RegexpParser.new(Regexp.new(source, options), { "dummy" => "dummy" })
+    regexp = parser.instance_variable_get("@regexp")
+    assert_equal(options, regexp.options)
+  end
+end