fluent-plugin-webhdfs 0.6.0rc1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e92e9863c1ee9908d2ee0076b3651b7198062511
-  data.tar.gz: b4cf49b2c59fd07f9cb5a3f7faecf165d60723b7
+  metadata.gz: 3001d6e76014cec6f043369f92db09fb9f6989d4
+  data.tar.gz: e7fdab6516fd7ef1ad2732369b002116e8b65b85
 SHA512:
-  metadata.gz: f9c1ea8130b845859ea7c9dd2504f03c176e6162d24636ca4ee27c12d36ac2e32dbcf962aaca77e394e3384d007a5328d974742ae258555130ff8ecf9384b2a6
-  data.tar.gz: 7afef45b797ba03cea394895dc3c6dd327b30339f825d3e2291dec5097b126dd803cd476e1b8420183141ff3f268da602623b0945a71c2f5261fb9b8a18062eb
+  metadata.gz: 898a6fa68810d9f94aefcb70e2b0e168b67b56922d2ae5f7757e1a8c9a38603d8ba0dc353ee49ae9247e42196fe3d8c88c31172773ac0df3919428bc25c84a5f
+  data.tar.gz: 37c6d7534dab627dd3a31bd4650f8642c983041cf1f06ff916cbc3107605d6070122ae8f1990cb36dd1f14ba19a3b2e5a65fbc0958553df4225c2228e81c29ca

data/.travis.yml

@@ -2,9 +2,10 @@ sudo: false
 language: ruby
 
 rvm:
+  - 2.0.0
   - 2.1
   - 2.2
-  - 2.3.1
+  - 2.3.0
 
 branches:
   only:
@@ -22,4 +23,12 @@ script: bundle exec rake test
 
 gemfile:
   - Gemfile
+  - gemfiles/fluentd_v0.12.gemfile
   - gemfiles/fluentd_v0.14.gemfile
+
+matrix:
+  exclude:
+    - rvm: 2.0.0
+      gemfile: Gemfile
+    - rvm: 2.0.0
+      gemfile: gemfiles/fluentd_v0.14.gemfile

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.6.0rc1"
+  gem.version       = "0.6.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -17,10 +17,10 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
-  gem.add_development_dependency "test-unit-rr"
   gem.add_development_dependency "appraisal"
   gem.add_development_dependency "snappy", '>= 0.0.13'
-  gem.add_runtime_dependency "fluentd", '>= 0.14.4'
+  gem.add_runtime_dependency "fluentd", ['>= 0.10.59', "< 0.14.0"]
+  gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
   gem.add_runtime_dependency "bzip2-ffi"
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -1,138 +1,129 @@
 # -*- coding: utf-8 -*-
 
-require 'fluent/plugin/output'
-require 'fluent/config/element'
-
-require 'webhdfs'
 require 'tempfile'
 require 'securerandom'
+require 'fluent/mixin/plaintextformatter'
 
-class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
+class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('webhdfs', self)
 
-  helpers :inject, :formatter, :compat_parameters
+  config_set_default :buffer_type, 'memory'
+  config_set_default :time_slice_format, '%Y%m%d'
+
+  # For fluentd v0.12.16 or earlier
+  class << self
+    unless method_defined?(:desc)
+      def desc(description)
+      end
+    end
+  end
 
   desc 'WebHDFS/HttpFs host'
-  config_param :host, :string, default: nil
+  config_param :host, :string, :default => nil
   desc 'WebHDFS/HttpFs port'
-  config_param :port, :integer, default: 50070
+  config_param :port, :integer, :default => 50070
   desc 'Namenode (host:port)'
-  config_param :namenode, :string, default: nil # host:port
+  config_param :namenode, :string, :default => nil # host:port
   desc 'Standby namenode for Namenode HA (host:port)'
-  config_param :standby_namenode, :string, default: nil # host:port
+  config_param :standby_namenode, :string, :default => nil # host:port
 
   desc 'Ignore errors on start up'
-  config_param :ignore_start_check_error, :bool, default: false
+  config_param :ignore_start_check_error, :bool, :default => false
 
   desc 'Output file path on HDFS'
   config_param :path, :string
   desc 'User name for pseudo authentication'
-  config_param :username, :string, default: nil
+  config_param :username, :string, :default => nil
 
   desc 'Store data over HttpFs instead of WebHDFS'
-  config_param :httpfs, :bool, default: false
+  config_param :httpfs, :bool, :default => false
 
   desc 'Number of seconds to wait for the connection to open'
-  config_param :open_timeout, :integer, default: 30 # from ruby net/http default
+  config_param :open_timeout, :integer, :default => 30 # from ruby net/http default
   desc 'Number of seconds to wait for one block to be read'
-  config_param :read_timeout, :integer, default: 60 # from ruby net/http default
+  config_param :read_timeout, :integer, :default => 60 # from ruby net/http default
 
   desc 'Retry automatically when known errors of HDFS are occurred'
-  config_param :retry_known_errors, :bool, default: false
+  config_param :retry_known_errors, :bool, :default => false
   desc 'Retry interval'
-  config_param :retry_interval, :integer, default: nil
+  config_param :retry_interval, :integer, :default => nil
   desc 'The number of retries'
-  config_param :retry_times, :integer, default: nil
+  config_param :retry_times, :integer, :default => nil
 
   # how many times of write failure before switch to standby namenode
   # by default it's 11 times that costs 1023 seconds inside fluentd,
   # which is considered enough to exclude the scenes that caused by temporary network fail or single datanode fail
   desc 'How many times of write failure before switch to standby namenode'
-  config_param :failures_before_use_standby, :integer, default: 11
+  config_param :failures_before_use_standby, :integer, :default => 11
+
+  include Fluent::Mixin::PlainTextFormatter
 
-  config_param :end_with_newline, :bool, default: true
+  config_param :default_tag, :string, :default => 'tag_missing'
 
   desc 'Append data or not'
-  config_param :append, :bool, default: true
+  config_param :append, :bool, :default => true
 
   desc 'Use SSL or not'
-  config_param :ssl, :bool, default: false
+  config_param :ssl, :bool, :default => false
   desc 'OpenSSL certificate authority file'
-  config_param :ssl_ca_file, :string, default: nil
+  config_param :ssl_ca_file, :string, :default => nil
   desc 'OpenSSL verify mode (none,peer)'
-  config_param :ssl_verify_mode, :enum, list: [:none, :peer], default: :none
+  config_param :ssl_verify_mode, :default => nil do |val|
+    case val
+    when 'none'
+      :none
+    when 'peer'
+      :peer
+    else
+      raise Fluent::ConfigError, "unexpected parameter on ssl_verify_mode: #{val}"
+    end
+  end
 
   desc 'Use kerberos authentication or not'
-  config_param :kerberos, :bool, default: false
+  config_param :kerberos, :bool, :default => false
 
-  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :text]
+  SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'snappy', 'lzo_command', 'text']
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
-  config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
-
-  config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
-  config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
-  config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
-  config_param :suppress_log_broken_string, :bool, default: false, deprecated: "use @log_level for plugin to suppress such info logs"
-
-  CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
-
-  config_section :buffer do
-    config_set_default :chunk_keys, ["time"]
+  config_param :compress, :default => nil do |val|
+    unless SUPPORTED_COMPRESS.include? val
+      raise Fluent::ConfigError, "unsupported compress: #{val}"
+    end
+    val
   end
 
-  config_section :format do
-    config_set_default :@type, 'out_file'
-    config_set_default :localtime, false # default timezone is UTC
-  end
+  CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
 
-  attr_reader :formatter, :compressor
+  attr_reader :compressor
 
   def initialize
     super
+    require 'net/http'
+    require 'time'
+    require 'webhdfs'
+
     @compressor = nil
-    @standby_namenode_host = nil
-    @output_include_tag = @output_include_time = nil # TODO: deprecated
-    @header_separator = @field_separator = nil # TODO: deprecated
   end
 
-  def configure(conf)
-    compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
 
-    timekey = case conf["path"]
-              when /%S/ then 1
-              when /%M/ then 60
-              when /%H/ then 3600
-              else 86400
-              end
-    if conf.elements(name: "buffer").empty?
-      e = Fluent::Config::Element.new("buffer", "time", {}, [])
-      conf.elements << e
+  def configure(conf)
+    if conf['path']
+      if conf['path'].index('%S')
+        conf['time_slice_format'] = '%Y%m%d%H%M%S'
+      elsif conf['path'].index('%M')
+        conf['time_slice_format'] = '%Y%m%d%H%M'
+      elsif conf['path'].index('%H')
+        conf['time_slice_format'] = '%Y%m%d%H'
+      end
     end
-    buffer_config = conf.elements(name: "buffer").first
-    buffer_config["timekey"] = timekey unless buffer_config["timekey"]
 
-    compat_parameters_convert_plaintextformatter(conf)
+    verify_config_placeholders_in_path!(conf)
 
     super
 
-    @formatter = formatter_create
-
-    if @using_formatter_config
-      @null_value = nil
-    else
-      @formatter.delimiter = "\x01" if @formatter.respond_to?(:delimiter) && @formatter.delimiter == 'SOH'
-      @null_value ||= 'NULL'
-    end
-
-    if @default_tag.nil? && !@using_formatter_config && @output_include_tag
-      @default_tag = "tag_missing"
-    end
-    if @remove_prefix
-      @remove_prefix_actual = @remove_prefix + "."
-      @remove_prefix_actual_length = @remove_prefix_actual.length
-    end
-
-    verify_config_placeholders_in_path!(conf)
     @replace_random_uuid = @path.include?('%{uuid}') || @path.include?('%{uuid_flush}')
     if @replace_random_uuid
       # to check SecureRandom.uuid is available or not (NotImplementedError raised in such environment)
@@ -143,7 +134,14 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       end
     end
 
-    @compressor = COMPRESSOR_REGISTRY.lookup(@compress.to_s).new
+    begin
+      @compressor = COMPRESSOR_REGISTRY.lookup(@compress || 'text').new
+    rescue Fluent::ConfigError
+      raise
+    rescue
+      $log.warn "#{@comress} not found. Use 'text' instead"
+      @compressor = COMPRESSOR_REGISTRY.lookup('text').new
+    end
 
     if @host
       @namenode_host = @host
@@ -178,7 +176,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       @client_standby = nil
     end
 
-    unless @append
+    if not @append
       if @path.index(CHUNK_ID_PLACE_HOLDER).nil?
         raise Fluent::ConfigError, "path must contain ${chunk_id}, which is the placeholder for chunk_id, when append is set to false."
       end
@@ -241,6 +239,14 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
   end
 
+  def shutdown
+    super
+  end
+
+  def path_format(chunk_key)
+    Time.strptime(chunk_key, @time_slice_format).strftime(@path)
+  end
+
   def is_standby_exception(e)
     e.is_a?(WebHDFS::IOError) && e.message.match(/org\.apache\.hadoop\.ipc\.StandbyException/)
   end
@@ -252,6 +258,12 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
   end
 
+  def chunk_unique_id_to_str(unique_id)
+    unique_id.unpack('C*').map{|x| x.to_s(16).rjust(2,'0')}.join('')
+  end
+
+  # TODO check conflictions
+
   def send_data(path, data)
     if @append
       begin
@@ -266,7 +278,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
 
   HOSTNAME_PLACEHOLDERS_DEPRECATED = ['${hostname}', '%{hostname}', '__HOSTNAME__']
   UUID_RANDOM_PLACEHOLDERS_DEPRECATED = ['${uuid}', '${uuid:random}', '__UUID__', '__UUID_RANDOM__']
-  UUID_OTHER_PLACEHOLDERS_OBSOLETED = ['${uuid:hostname}', '%{uuid:hostname}', '__UUID_HOSTNAME__', '${uuid:timestamp}', '%{uuid:timestamp}', '__UUID_TIMESTAMP__']
+  UUID_OTHER_PLACEHOLDERS_OBSOLETED = ['${uuid:hostname}', '%{uuid:hostname}', '__UUID_HOSTNAME__', '${uuid:timestamp}', '%{uuid:timestamp}', '__UUID_TIMESTAMP__']  
 
   def verify_config_placeholders_in_path!(conf)
     return unless conf.has_key?('path')
@@ -295,20 +307,20 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
           log.error "configuration placeholder #{ph} is now unsupported by webhdfs output plugin."
         end
       end
-      raise ConfigError, "there are unsupported placeholders in path."
+      raise Fluent::ConfigError, "there are unsupported placeholders in path."
     end
   end
 
   def generate_path(chunk)
     hdfs_path = if @append
-                  extract_placeholders(@path, chunk.metadata)
+                  path_format(chunk.key)
                 else
-                  extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id(chunk.unique_id))
+                  path_format(chunk.key).gsub(CHUNK_ID_PLACE_HOLDER, chunk_unique_id_to_str(chunk.unique_id))
                 end
     hdfs_path = "#{hdfs_path}#{@compressor.ext}"
     if @replace_random_uuid
       uuid_random = SecureRandom.uuid
-      hdfs_path.gsub!('%{uuid}', uuid_random).gsub!('%{uuid_flush}', uuid_random)
+      hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)
     end
     hdfs_path
   end
@@ -324,48 +336,6 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
   end
 
-  def format(tag, time, record)
-    if @remove_prefix # TODO: remove when it's obsoleted
-      if tag.start_with?(@remove_prefix_actual)
-        if tag.length > @remove_prefix_actual_length
-          tag = tag[@remove_prefix_actual_length..-1]
-        else
-          tag = @default_tag
-        end
-      elsif tag.start_with?(@remove_prefix)
-        if tag == @remove_prefix
-          tag = @default_tag
-        else
-          tag = tag.sub(@remove_prefix, '')
-        end
-      end
-    end
-
-    if @null_value # TODO: remove when it's obsoleted
-      check_keys = (record.keys + @null_convert_keys).uniq
-      check_keys.each do |key|
-        record[key] = @null_value if record[key].nil?
-      end
-    end
-
-    if @using_formatter_config
-      record = inject_values_to_record(tag, time, record)
-      line = @formatter.format(tag, time, record)
-    else # TODO: remove when it's obsoleted
-      time_str = @output_include_time ? @time_formatter.call(time) + @header_separator : ''
-      tag_str = @output_include_tag ? tag + @header_separator : ''
-      record_str = @formatter.format(tag, time, record)
-      line = time_str + tag_str + record_str
-    end
-    line << "\n" if @end_with_newline && !line.end_with?("\n")
-    line
-  rescue => e # remove this clause when @suppress_log_broken_string is obsoleted
-    unless @suppress_log_broken_string
-      log.info "unexpected error while formatting events, ignored", tag: tag, record: record, error: e
-    end
-    ''
-  end
-
   def write(chunk)
     hdfs_path = generate_path(chunk)
 
@@ -396,72 +366,6 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     hdfs_path
   end
 
-  def compat_parameters_convert_plaintextformatter(conf)
-    if !conf.elements('format').empty? || !conf['output_data_type']
-      @using_formatter_config = true
-      @null_convert_keys = []
-      return
-    end
-
-    log.warn "webhdfs output plugin is working with old configuration parameters. use <inject>/<format> sections instead for further releases."
-    @using_formatter_config = false
-    @null_convert_keys = []
-
-    @header_separator = case conf['field_separator']
-                        when nil     then "\t"
-                        when 'SPACE' then ' '
-                        when 'TAB'   then "\t"
-                        when 'COMMA' then ','
-                        when 'SOH'   then "\x01"
-                        else conf['field_separator']
-                        end
-
-    format_section = Fluent::Config::Element.new('format', '', {}, [])
-    case conf['output_data_type']
-    when '', 'json' # blank value is for compatibility reason (especially in testing)
-      format_section['@type'] = 'json'
-    when 'ltsv'
-      format_section['@type'] = 'ltsv'
-    else
-      unless conf['output_data_type'].start_with?('attr:')
-        raise Fluent::ConfigError, "output_data_type is invalid: #{conf['output_data_type']}"
-      end
-      format_section['@format'] = 'tsv'
-      keys_part = conf['output_data_type'].sub(/^attr:/, '')
-      @null_convert_keys = keys_part.split(',')
-      format_section['keys'] = keys_part
-      format_section['delimiter'] = case conf['field_separator']
-                                    when nil then '\t'
-                                    when 'SPACE' then ' '
-                                    when 'TAB'   then '\t'
-                                    when 'COMMA' then ','
-                                    when 'SOH'   then 'SOH' # fixed later
-                                    else conf['field_separator']
-                                    end
-    end
-
-    conf.elements << format_section
-
-    @output_include_time = conf.has_key?('output_include_time') ? Fluent::Config.bool_value(conf['output_include_time']) : true
-    @output_include_tag = conf.has_key?('output_include_tag') ? Fluent::Config.bool_value(conf['output_include_tag']) : true
-
-    if @output_include_time
-      # default timezone is UTC
-      using_localtime = if !conf.has_key?('utc') && !conf.has_key?('localtime')
-                          false
-                        elsif conf.has_key?('localtime') && conf.has_key?('utc')
-                          raise Fluent::ConfigError, "specify either 'localtime' or 'utc'"
-                        elsif conf.has_key?('localtime')
-                          Fluent::Config.bool_value('localtime')
-                        else
-                          Fluent::Config.bool_value('utc')
-                        end
-      @time_formatter = Fluent::TimeFormatter.new(conf['time_format'], using_localtime)
-    else
-      @time_formatter = nil
-    end
-  end
-
   class Compressor
     include Fluent::Configurable
 
@@ -488,7 +392,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       begin
         Open3.capture3("#{command} -V")
       rescue Errno::ENOENT
-        raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+        raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
       end
     end
   end

data/lib/fluent/plugin/webhdfs_compressor_bzip2.rb

@@ -1,5 +1,5 @@
-module Fluent::Plugin
-  class WebHDFSOutput < Output
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
     class Bzip2Compressor < Compressor
       WebHDFSOutput.register_compressor('bzip2', self)
 

data/lib/fluent/plugin/webhdfs_compressor_gzip.rb

@@ -1,5 +1,5 @@
-module Fluent::Plugin
-  class WebHDFSOutput < Output
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
     class GzipCompressor < Compressor
       WebHDFSOutput.register_compressor('gzip', self)
 

data/lib/fluent/plugin/webhdfs_compressor_lzo_command.rb

@@ -1,9 +1,9 @@
-module Fluent::Plugin
-  class WebHDFSOutput < Output
+module Fluent
+  class WebHDFSOutput
     class LZOCommandCompressor < Compressor
       WebHDFSOutput.register_compressor('lzo_command', self)
 
-      config_param :command_parameter, :string, default: '-qf1'
+      config_param :command_parameter, :string, :default => '-qf1'
 
       def configure(conf)
         super

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -1,5 +1,5 @@
-module Fluent::Plugin
-  class WebHDFSOutput < Output
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
     class SnappyCompressor < Compressor
       WebHDFSOutput.register_compressor('snappy', self)
 

data/lib/fluent/plugin/webhdfs_compressor_text.rb

@@ -1,5 +1,5 @@
-module Fluent::Plugin
-  class WebHDFSOutput < Output
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
     class TextCompressor < Compressor
       WebHDFSOutput.register_compressor('text', self)
 

data/test/helper.rb

@@ -8,13 +8,10 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'test/unit'
-require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
-require 'fluent/test/helpers'
-require 'fluent/test/driver/output'
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|
@@ -25,8 +22,6 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-include Fluent::Test::Helpers
-
 require 'fluent/plugin/out_webhdfs'
 
 class Test::Unit::TestCase

data/test/plugin/test_compressor.rb

@@ -16,11 +16,11 @@ class CompressorTest < Test::Unit::TestCase
     def setup
       omit unless Object.const_defined?(:Snappy)
       Fluent::Test.setup
-      @compressor = Fluent::Plugin::WebHDFSOutput::SnappyCompressor.new
+      @compressor = Fluent::WebHDFSOutput::SnappyCompressor.new
     end
 
-    def create_driver(conf = CONFIG)
-      Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
+    def create_driver(conf=CONFIG,tag='test')
+      Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
     end
 
     def test_ext

data/test/plugin/test_out_webhdfs.rb

@@ -1,243 +1,166 @@
 require 'helper'
 
 class WebHDFSOutputTest < Test::Unit::TestCase
-  CONFIG_DEFAULT = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d.log"})
-
-  CONFIG_COMPAT = config_element(
-    "ROOT", "", {
-      "output_data_type" => "",
-      "host" => "namenode.local",
-      "path" => "/hdfs/path/file.%Y%m%d.log"
-    })
-
+  CONFIG = %[
+host namenode.local
+path /hdfs/path/file.%Y%m%d.log
+  ]
   def setup
     Fluent::Test.setup
   end
 
-  def create_driver(conf)
-    Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
   end
 
-  sub_test_case "default configuration" do
-    test 'configured with standard out_file format with specified hdfs info' do
-      d = create_driver(CONFIG_DEFAULT)
-      assert_true d.instance.instance_eval{ @using_formatter_config }
-
+  class ConfigureTest < self
+    def test_default
+      d = create_driver
       assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 50070, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+      assert_equal '%Y%m%d', d.instance.time_slice_format
       assert_equal false, d.instance.httpfs
       assert_nil d.instance.username
       assert_equal false, d.instance.ignore_start_check_error
 
-      assert_equal 'Fluent::Plugin::OutFileFormatter', d.instance.formatter.class.to_s
-      assert_equal true, d.instance.end_with_newline
-
-      # deprecated params
-      assert_nil d.instance.instance_eval{ @output_include_time }
-      assert_nil d.instance.instance_eval{ @output_include_tag }
+      assert_equal true, d.instance.output_include_time
+      assert_equal true, d.instance.output_include_tag
+      assert_equal 'json', d.instance.output_data_type
       assert_nil d.instance.remove_prefix
-      assert_nil d.instance.instance_eval{ @header_separator }
-      assert_nil d.instance.default_tag
-    end
-  end
-
-  sub_test_case "flat configuration" do
-    def test_default_for_traditional_config
-      d = create_driver(CONFIG_COMPAT)
-      assert_false d.instance.instance_eval{ @using_formatter_config }
-
-      assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
-      assert_equal 50070, d.instance.instance_eval{ @namenode_port }
-      assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
-      assert_equal false, d.instance.httpfs
-      assert_nil d.instance.username
-      assert_equal false, d.instance.ignore_start_check_error
-
-      assert_equal 'Fluent::Plugin::JSONFormatter', d.instance.formatter.class.to_s
-      assert_equal true, d.instance.end_with_newline
-
-      assert_equal true, d.instance.instance_eval{ @output_include_time }
-      assert_equal true, d.instance.instance_eval{ @output_include_tag }
-      assert_nil d.instance.instance_eval{ @remove_prefix }
-      assert_equal "\t", d.instance.instance_eval{ @header_separator }
-      assert_equal 'tag_missing', d.instance.instance_eval{ @default_tag }
+      assert_equal 'TAB', d.instance.field_separator
+      assert_equal true, d.instance.add_newline
+      assert_equal 'tag_missing', d.instance.default_tag
     end
 
     def test_httpfs
-      conf = config_element(
-        "ROOT", "", {
-          "namenode" => "server.local:14000",
-          "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
-          "httpfs" => "yes",
-          "username" => "hdfs_user"
-        })
-      d = create_driver(conf)
-
+      d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+httpfs yes
+username hdfs_user
+]
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
       assert_equal true, d.instance.httpfs
       assert_equal 'hdfs_user', d.instance.username
     end
 
     def test_ssl
-      conf = config_element(
-        "ROOT", "", {
-          "namenode" => "server.local:14000",
-          "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
-          "ssl" => true,
-          "ssl_ca_file" => "/path/to/ca_file.pem",
-          "ssl_verify_mode" => "peer",
-          "kerberos" => true
-        })
-      d = create_driver(conf)
-
+      d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+ssl true
+ssl_ca_file /path/to/ca_file.pem
+ssl_verify_mode peer
+kerberos true
+]
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
       assert_equal true, d.instance.ssl
       assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
       assert_equal :peer, d.instance.ssl_verify_mode
       assert_equal true, d.instance.kerberos
     end
 
-    data(gzip: [:gzip, Fluent::Plugin::WebHDFSOutput::GzipCompressor],
-         bzip2: [:bzip2, Fluent::Plugin::WebHDFSOutput::Bzip2Compressor],
-         snappy: [:snappy, Fluent::Plugin::WebHDFSOutput::SnappyCompressor],
-         lzo: [:lzo_command, Fluent::Plugin::WebHDFSOutput::LZOCommandCompressor])
+    data(gzip: ['gzip', Fluent::WebHDFSOutput::GzipCompressor],
+         bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor],
+         snappy: ['snappy', Fluent::WebHDFSOutput::SnappyCompressor],
+         lzo: ['lzo_command', Fluent::WebHDFSOutput::LZOCommandCompressor])
     def test_compress(data)
       compress_type, compressor_class = data
       begin
-        conf = config_element(
-          "ROOT", "", {
-            "namenode" => "server.local:14000",
-            "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
-            "compress" => compress_type
-          })
-        d = create_driver(conf)
+        d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+compress #{compress_type}
+        ]
       rescue Fluent::ConfigError => ex
         omit ex.message
       end
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
       assert_equal compress_type, d.instance.compress
       assert_equal compressor_class, d.instance.compressor.class
     end
 
-    def test_placeholders_old_style
-      conf = config_element(
-        "ROOT", "", {
-          "hostname" => "testing.node.local",
-          "namenode" => "server.local:50070",
-          "path" => "/hdfs/${hostname}/file.%Y%m%d%H.log"
-        })
-      d = create_driver(conf)
+    def test_placeholders
+      d = create_driver %[
+hostname testing.node.local
+namenode server.local:50070
+path /hdfs/${hostname}/file.%Y%m%d%H.log
+]
       assert_equal '/hdfs/testing.node.local/file.%Y%m%d%H.log', d.instance.path
     end
 
-    data("%Y%m%d" => ["/hdfs/path/file.%Y%m%d.log", "/hdfs/path/file.20120718.log"],
-         "%Y%m%d.%H%M" => ["/hdfs/path/file.%Y%m%d.%H%M.log", "/hdfs/path/file.20120718.1503.log"])
-    test "generate_path" do |(path, expected)|
-      conf = config_element(
-        "ROOT", "", {
-          "namenode" => "server.local:14000",
-          "path" => path
-        })
-      d = create_driver(conf)
-      formatter = Fluent::Timezone.formatter("+0900", path)
-      mock(Fluent::Timezone).formatter(Time.now.strftime("%z"), path) { formatter }
-      time = event_time("2012-07-18 15:03:00 +0900")
-      metadata = d.instance.metadata("test", time, {})
-      chunk = d.instance.buffer.generate_chunk(metadata)
-      assert_equal expected, d.instance.generate_path(chunk)
-    end
-
-    data(path: { "append" => false },
-         ssl: { "ssl" => true, "ssl_verify_mode" => "invalid" },
-         compress: { "compress" => "invalid" })
-    test "invalid" do |attr|
-      conf = config_element(
-        "ROOT", "", {
-          "namenode" => "server.local:14000",
-          "path" => "/hdfs/path/file.%Y%m%d.%H%M.log"
-        })
-      conf += config_element("", "", attr)
-      assert_raise Fluent::ConfigError do
-        create_driver(conf)
+    class PathFormatTest < self
+      def test_default
+        d = create_driver
+        assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+        assert_equal '%Y%m%d', d.instance.time_slice_format
+        assert_equal '/hdfs/path/file.20120718.log', d.instance.path_format('20120718')
       end
-    end
-  end
-
-  sub_test_case "sub section configuration" do
-    def test_time_key
-      conf = config_element(
-        "ROOT", "", {
-          "host" => "namenode.local",
-          "path" => "/hdfs/path/file.%Y%m%d.log"
-        }, [
-          config_element(
-            "buffer", "time", {
-              "timekey" => 1
-            })
-        ]
-      )
-      d = create_driver(conf)
-      time = event_time("2012-07-18 15:03:00 +0900")
-      metadata = d.instance.metadata("test", time, {})
-      chunk = d.instance.buffer.generate_chunk(metadata)
-      assert_equal 1, d.instance.buffer_config.timekey
-      assert_equal "/hdfs/path/file.20120718.log", d.instance.generate_path(chunk)
-    end
-  end
 
-  sub_test_case "using format subsection" do
-    test "blank format means default format 'out_file' with UTC timezone" do
-      format_section = config_element("format", "", {}, [])
-      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [format_section])
-      d = create_driver(conf)
-      time = event_time("2017-01-24 13:10:30 -0700")
-      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
-      assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
+      def test_time_slice_format
+        d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+]
+        assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+        assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+        assert_equal '/hdfs/path/file.20120718.1503.log', d.instance.path_format('201207181503')
+      end
     end
 
-    test "specifying timezone works well in format section" do
-      format_section = config_element("format", "", {"timezone" => "+0100"}, [])
-      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [format_section])
-      d = create_driver(conf)
-      time = event_time("2017-01-24 13:10:30 -0700")
-      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
-      assert_equal "2017-01-24T21:10:30+01:00\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
-    end
+    class InvalidTest < self
+      def test_path
+        assert_raise Fluent::ConfigError do
+          d = create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/file.%Y%m%d.%H%M.log
+            append false
+          ]
+        end
+      end
 
-    test "specifying formatter type LTSV for records, without tag and timezone" do
-      format_section = config_element("format", "", {"@type" => "ltsv"}, [])
-      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [format_section])
-      d = create_driver(conf)
-      time = event_time("2017-01-24 13:10:30 -0700")
-      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
-      assert_equal "message:yay\tname:tagomoris\n", line
-    end
+      def test_ssl
+        assert_raise Fluent::ConfigError do
+          create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/file.%Y%m%d.%H%M.log
+            ssl true
+            ssl_verify_mode invalid
+          ]
+        end
+      end
 
-    test "specifying formatter type LTSV for records, with inject section to insert tag and time" do
-      inject_section = config_element("inject", "", {"tag_key" => "tag", "time_key" => "time", "time_type" => "string", "localtime" => "false"})
-      format_section = config_element("format", "", {"@type" => "ltsv"}, [])
-      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [inject_section, format_section])
-      d = create_driver(conf)
-      time = event_time("2017-01-24 13:10:30 -0700")
-      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
-      assert_equal "message:yay\tname:tagomoris\ttag:test.now\ttime:2017-01-24T20:10:30Z\n", line
-    end
-  end
+      def test_invalid_compress
+        assert_raise Fluent::ConfigError do
+          create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/file.%Y%m%d.%H%M.log
+            compress invalid
+          ]
+        end
+      end
 
-  sub_test_case "using older configuration" do
-    test "output_data_type json is same with out_file with UTC timezone" do
-      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log", "output_data_type" => "json"}, [])
-      d = create_driver(conf)
-      time = event_time("2017-01-24 13:10:30 -0700")
-      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
-      assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
+      data("%{uuid:hostname}" => "%{uuid:hostname}",
+           "%{uuid:timestamp}" => "%{uuid:timestamp}")
+      def test_obsoleted_placeholders(placeholder)
+        assert_raise Fluent::ConfigError do
+          create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/#{placeholder}/file.%Y%m%d.%H%M.log
+            append false
+          ]
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.6.0rc1
+  version: 0.6.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-24 00:00:00.000000000 Z
+date: 2017-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: test-unit-rr
+  name: appraisal
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,47 +53,53 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: appraisal
+  name: snappy
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.13
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.13
 - !ruby/object:Gem::Dependency
-  name: snappy
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.13
-  type: :development
+        version: 0.10.59
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 0.10.59
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: fluent-mixin-plaintextformatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.4
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.4
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -163,12 +169,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting