fluentd 0.12.40 → 1.6.2

data/templates/new_gem/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/templates/new_gem/fluent-plugin.gemspec.erb

@@ -0,0 +1,27 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "<%= gem_name %>"
+  spec.version = "0.1.0"
+  spec.authors = ["<%= user_name %>"]
+  spec.email   = ["<%= user_email %>"]
+
+  spec.summary       = %q{TODO: Write a short summary, because Rubygems requires one.}
+  spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = "TODO: Put your gem's website or public repo URL here."
+  spec.license       = "<%= @license.name %>"
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end

data/templates/new_gem/lib/fluent/plugin/filter.rb.erb

@@ -0,0 +1,14 @@
+<%= preamble %>
+
+require "fluent/plugin/filter"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Filter
+      Fluent::Plugin.register_filter("<%= plugin_name %>", self)
+
+      def filter(tag, time, record)
+      end
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb

@@ -0,0 +1,14 @@
+<%= preamble %>
+
+require "fluent/plugin/formatter"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Formatter
+      Fluent::Plugin.register_formatter("<%= name %>", self)
+
+      def format(tag, time, record)
+      end
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/input.rb.erb

@@ -0,0 +1,11 @@
+<%= preamble %>
+
+require "fluent/plugin/input"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Input
+      Fluent::Plugin.register_input("<%= plugin_name %>", self)
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/output.rb.erb

@@ -0,0 +1,11 @@
+<%= preamble %>
+
+require "fluent/plugin/output"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Output
+      Fluent::Plugin.register_output("<%= plugin_name %>", self)
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/parser.rb.erb

@@ -0,0 +1,15 @@
+<%= preamble %>
+
+require "fluent/plugin/parser"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Parser
+      Fluent::Plugin.register_parser("<%= plugin_name %>", self)
+
+      def parse(text)
+        yield time, record
+      end
+    end
+  end
+end

data/templates/new_gem/test/helper.rb.erb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/<%= type %>"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/templates/new_gem/test/plugin/test_filter.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end

data/templates/new_gem/test/plugin/test_formatter.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end

data/templates/new_gem/test/plugin/test_input.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end

data/templates/new_gem/test/plugin/test_output.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end

data/templates/new_gem/test/plugin/test_parser.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end

data/templates/plugin_config_formatter/param.md-compact.erb

@@ -0,0 +1,25 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+-%>
+* **<%= name %>** (<%= type %>) (<%= required_label %>): <%= description %>
+<%- if type == :enum -%>
+  * Available values: <%= config[:list].join(", ") %>
+<%- end -%>
+<%- if default -%>
+  * Default value: `<%= default %>`.
+<%- end -%>
+<%- if alias_name -%>
+  * Alias: <%= alias_name %>
+<%- end -%>
+<%- if deprecated -%>
+  * Deprecated: <%= deprecated %>
+<%- end -%>
+<%- if obsoleted -%>
+  * Obsoleted: <%= :obsoleted %>
+<%- end -%>

data/templates/plugin_config_formatter/param.md.erb

@@ -0,0 +1,34 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+param_header = "#" * (3 + level)
+-%>
+<%= param_header %> <%= name %> (<%= type %>) (<%= required_label %>)
+
+<%= description %>
+<%- if type == :enum -%>
+
+Available values: <%= config[:list].join(", ") %>
+<%- end -%>
+<%- if default -%>
+
+Default value: `<%= default %>`.
+<%- end -%>
+<%- if alias_name -%>
+
+Alias: <%= alias_name %>
+<%- end -%>
+<%- if deprecated -%>
+
+Deprecated: <%= deprecated %>
+<%- end -%>
+<%- if obsoleted -%>
+
+Obsoleted: <%= :obsoleted %>
+<%- end -%>
+

data/templates/plugin_config_formatter/section.md.erb

@@ -0,0 +1,12 @@
+<%-
+section_header = "#" * (3 + level)
+required_label = required ? "required" : "optional"
+multiple_label = multi ? "multiple" : "single"
+-%>
+<%= section_header %> \<<%= section_name %>\> section (<%= required_label %>) (<%= multiple_label %>)
+<%- if alias_name -%>
+
+Alias: <%= alias_name %>
+<%- end -%>
+
+<%= dump_section_markdown(sub_section, level + 1) %>

data/test/command/test_binlog_reader.rb

@@ -0,0 +1,346 @@
+require_relative '../helper'
+
+require 'yajl'
+require 'flexmock/test_unit'
+
+require 'fluent/command/binlog_reader'
+require 'fluent/event'
+
+class TestFluentBinlogReader < ::Test::Unit::TestCase
+  module ::BinlogReaderCommand
+    class Dummy < Base
+      def call; end
+    end
+  end
+
+  def suppress_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+  ensure
+    $stdout = STDOUT
+  end
+
+  sub_test_case 'call' do
+    data(
+      empty: [],
+      invalid: %w(invalid packed.log),
+    )
+    test 'should fail when invalid command' do |argv|
+      fu = FluentBinlogReader.new(argv)
+
+      assert_raise(SystemExit) do
+        suppress_stdout { fu.call }
+      end
+    end
+
+    data(
+      cat: %w(cat packed.log),
+      head: %w(head packed.log),
+      formats: %w(formats packed.log)
+    )
+    test 'should succeed when valid command' do |argv|
+      fu = FluentBinlogReader.new(argv)
+
+      flexstub(::BinlogReaderCommand) do |command|
+        command.should_receive(:const_get).once.and_return(::BinlogReaderCommand::Dummy)
+        assert_nothing_raised do
+          fu.call
+        end
+      end
+    end
+  end
+end
+
+class TestBaseCommand < ::Test::Unit::TestCase
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/binlog_reader#{ENV['TEST_ENV_NUMBER']}")
+
+  def create_message_packed_file(path, times = [event_time], records = [{ 'message' => 'dummy' }])
+    es = Fluent::MultiEventStream.new(times, records)
+    v = es.to_msgpack_stream
+    out_path = "#{TMP_DIR}/#{path}"
+    File.open(out_path, 'wb') do |f|
+      f.print(v)
+    end
+    waiting(5) do
+      sleep 0.5 until File.size(out_path) == v.bytesize
+    end
+  end
+
+  def setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  def timezone(timezone = 'UTC')
+    old = ENV['TZ']
+    ENV['TZ'] = timezone
+    yield
+  ensure
+    ENV['TZ'] = old
+  end
+end
+
+class TestHead < TestBaseCommand
+  sub_test_case 'initialize' do
+    data(
+      'file is not passed' => %w(),
+      'file is not found' => %w(invalid_path.log)
+    )
+    test 'should fail if file is invalid' do |argv|
+      assert_raise(SystemExit) do
+        capture_stdout { BinlogReaderCommand::Head.new(argv) }
+      end
+    end
+
+    test 'should succeed if a file is valid' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}"]
+      create_message_packed_file(file_name)
+
+      assert_nothing_raised do
+        BinlogReaderCommand::Head.new(argv)
+      end
+    end
+
+    test 'should fail when config_params format is invalid' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'only_key']
+      create_message_packed_file(file_name)
+
+      assert_raise(SystemExit) do
+        capture_stdout { BinlogReaderCommand::Head.new(argv) }
+      end
+    end
+
+    test 'should succeed if config_params format is valid' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message']
+      create_message_packed_file(file_name)
+
+      assert_nothing_raised do
+        capture_stdout { BinlogReaderCommand::Head.new(argv) }
+      end
+    end
+  end
+
+  sub_test_case 'call' do
+    setup do
+      @file_name = 'packed.log'
+      @t = '2011-01-02 13:14:15 UTC'
+      @record = { 'message' => 'dummy' }
+    end
+
+    test 'should output the beginning of the file with default format (out_file)' do
+      argv = ["#{TMP_DIR}/#{@file_name}"]
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
+        head = BinlogReaderCommand::Head.new(argv)
+        out = capture_stdout { head.call }
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 5, out
+      end
+    end
+
+    test 'should set the number of lines to display' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '-n', '1']
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
+        head = BinlogReaderCommand::Head.new(argv)
+        out = capture_stdout { head.call }
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+      end
+    end
+
+    test 'should fail when the number of lines is invalid' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '-n', '0']
+
+      create_message_packed_file(@file_name)
+      assert_raise(SystemExit) do
+        capture_stdout { BinlogReaderCommand::Head.new(argv) }
+      end
+    end
+
+    test 'should output content of a file with json format' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '--format=json']
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
+        head = BinlogReaderCommand::Head.new(argv)
+        out = capture_stdout { head.call }
+        assert_equal "#{Yajl.dump(@record)}\n", out
+      end
+    end
+
+    test 'should fail with an invalid format' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '--format=invalid']
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
+        head = BinlogReaderCommand::Head.new(argv)
+
+        assert_raise(SystemExit) do
+          capture_stdout { head.call }
+        end
+      end
+    end
+
+    test 'should succeed if multiple config_params format' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message,fo', '-e', 'delimiter=|']
+      create_message_packed_file(file_name, [event_time], [{ 'message' => 'dummy', 'fo' => 'dummy2' }])
+
+      head = BinlogReaderCommand::Head.new(argv)
+      assert_equal "\"dummy\"|\"dummy2\"\n", capture_stdout { head.call }
+    end
+  end
+end
+
+class TestCat < TestBaseCommand
+  sub_test_case 'initialize' do
+    data(
+      'file is not passed' => [],
+      'file is not found' => %w(invalid_path.log)
+    )
+    test 'should fail if a file is invalid' do |argv|
+      assert_raise(SystemExit) do
+        capture_stdout { BinlogReaderCommand::Head.new(argv) }
+      end
+    end
+
+    test 'should succeed if a file is valid' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}"]
+      create_message_packed_file(file_name)
+
+      assert_nothing_raised do
+        BinlogReaderCommand::Cat.new(argv)
+      end
+    end
+
+    test 'should fail when config_params format is invalid' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}", '--format=json', '-e', 'only_key']
+      create_message_packed_file(file_name)
+
+      assert_raise(SystemExit) do
+        capture_stdout { BinlogReaderCommand::Cat.new(argv) }
+      end
+    end
+
+    test 'should succeed when config_params format is valid' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message']
+      create_message_packed_file(file_name)
+
+      assert_nothing_raised do
+        capture_stdout { BinlogReaderCommand::Cat.new(argv) }
+      end
+    end
+  end
+
+  sub_test_case 'call' do
+    setup do
+      @file_name = 'packed.log'
+      @t = '2011-01-02 13:14:15 UTC'
+      @record = { 'message' => 'dummy' }
+    end
+
+    test 'should output the file with default format(out_file)' do
+      argv = ["#{TMP_DIR}/#{@file_name}"]
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
+        head = BinlogReaderCommand::Cat.new(argv)
+        out = capture_stdout { head.call }
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 6, out
+      end
+    end
+
+    test 'should set the number of lines to display' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '-n', '1']
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
+        head = BinlogReaderCommand::Cat.new(argv)
+        out = capture_stdout { head.call }
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+      end
+    end
+
+    test 'should output content of a file with json format' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '--format=json']
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
+        head = BinlogReaderCommand::Cat.new(argv)
+        out = capture_stdout { head.call }
+        assert_equal "#{Yajl.dump(@record)}\n", out
+      end
+    end
+
+    test 'should fail with an invalid format' do
+      argv = ["#{TMP_DIR}/#{@file_name}", '--format=invalid']
+
+      timezone do
+        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
+        head = BinlogReaderCommand::Cat.new(argv)
+
+        assert_raise(SystemExit) do
+          capture_stdout { head.call }
+        end
+      end
+    end
+
+    test 'should succeed if multiple config_params format' do
+      file_name = 'packed.log'
+      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message,fo', '-e', 'delimiter=|']
+      create_message_packed_file(file_name, [event_time], [{ 'message' => 'dummy', 'fo' => 'dummy2' }])
+
+      head = BinlogReaderCommand::Cat.new(argv)
+      assert_equal "\"dummy\"|\"dummy2\"\n", capture_stdout { head.call }
+    end
+  end
+end
+
+class TestFormats < TestBaseCommand
+  test 'parse_option!' do
+    assert_raise(SystemExit) do
+      capture_stdout do
+        BinlogReaderCommand::Formats.new(['--plugin=invalid_dir_path'])
+      end
+    end
+  end
+
+  sub_test_case 'call' do
+    test 'display available plugins' do
+      f = BinlogReaderCommand::Formats.new
+      out = capture_stdout { f.call }
+      assert out.include?('json')
+      assert out.include?('csv')
+    end
+
+    test 'add new plugins using --plugin option' do
+      dir_path = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter1')
+
+      f = BinlogReaderCommand::Formats.new(["--plugin=#{dir_path}"])
+      out = capture_stdout { f.call }
+      assert out.include?('json')
+      assert out.include?('csv')
+      assert out.include?('test1')
+    end
+
+    test 'add multiple plugins using --plugin option' do
+      dir_path1 = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter1')
+      dir_path2 = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter2')
+
+      f = BinlogReaderCommand::Formats.new(["--plugin=#{dir_path1}", '-p', dir_path2])
+      out = capture_stdout { f.call }
+      assert out.include?('json')
+      assert out.include?('csv')
+      assert out.include?('test1')
+      assert out.include?('test2')
+    end
+  end
+end