fluent-plugin-webhdfs 0.5.3 → 0.6.0rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 81d8b4440f1fcf4a086791d26527cb0b8ef86cc4
-  data.tar.gz: 99b1b82fca276cd88da3af26ef07843c6d3e6751
+  metadata.gz: e92e9863c1ee9908d2ee0076b3651b7198062511
+  data.tar.gz: b4cf49b2c59fd07f9cb5a3f7faecf165d60723b7
 SHA512:
-  metadata.gz: 2b41f320743c71ac0cb14e4599a2111cde800237c9512885966e25a041231bd70472497467d994cc67865b432b86cca2b959d48185edad80333dd471d5b489ca
-  data.tar.gz: d0ade845373ff150904de373480bdeb2fcfe01ad8e376e6f4850f233d25d5094a544a35fc4057c2a54588d85c122d6e63265156fda7397c1a15890d0f43b294e
+  metadata.gz: f9c1ea8130b845859ea7c9dd2504f03c176e6162d24636ca4ee27c12d36ac2e32dbcf962aaca77e394e3384d007a5328d974742ae258555130ff8ecf9384b2a6
+  data.tar.gz: 7afef45b797ba03cea394895dc3c6dd327b30339f825d3e2291dec5097b126dd803cd476e1b8420183141ff3f268da602623b0945a71c2f5261fb9b8a18062eb

data/.travis.yml

@@ -2,10 +2,9 @@ sudo: false
 language: ruby
 
 rvm:
-  - 2.0.0
   - 2.1
   - 2.2
-  - 2.3.0
+  - 2.3.1
 
 branches:
   only:
@@ -23,12 +22,4 @@ script: bundle exec rake test
 
 gemfile:
   - Gemfile
-  - gemfiles/fluentd_v0.12.gemfile
   - gemfiles/fluentd_v0.14.gemfile
-
-matrix:
-  exclude:
-    - rvm: 2.0.0
-      gemfile: Gemfile
-    - rvm: 2.0.0
-      gemfile: gemfiles/fluentd_v0.14.gemfile

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.5.3"
+  gem.version       = "0.6.0rc1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -17,11 +17,10 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "test-unit-rr"
   gem.add_development_dependency "appraisal"
   gem.add_development_dependency "snappy", '>= 0.0.13'
-  gem.add_runtime_dependency "fluentd", ['>= 0.10.59', "< 0.14.0"]
-  gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
-  gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
+  gem.add_runtime_dependency "fluentd", '>= 0.14.4'
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
   gem.add_runtime_dependency "bzip2-ffi"
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -1,139 +1,150 @@
 # -*- coding: utf-8 -*-
 
-require 'tempfile'
+require 'fluent/plugin/output'
+require 'fluent/config/element'
 
-require 'fluent/mixin/config_placeholders'
-require 'fluent/mixin/plaintextformatter'
+require 'webhdfs'
+require 'tempfile'
+require 'securerandom'
 
-class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
+class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('webhdfs', self)
 
-  config_set_default :buffer_type, 'memory'
-  config_set_default :time_slice_format, '%Y%m%d'
-
-  # For fluentd v0.12.16 or earlier
-  class << self
-    unless method_defined?(:desc)
-      def desc(description)
-      end
-    end
-  end
+  helpers :inject, :formatter, :compat_parameters
 
   desc 'WebHDFS/HttpFs host'
-  config_param :host, :string, :default => nil
+  config_param :host, :string, default: nil
   desc 'WebHDFS/HttpFs port'
-  config_param :port, :integer, :default => 50070
+  config_param :port, :integer, default: 50070
   desc 'Namenode (host:port)'
-  config_param :namenode, :string, :default => nil # host:port
+  config_param :namenode, :string, default: nil # host:port
   desc 'Standby namenode for Namenode HA (host:port)'
-  config_param :standby_namenode, :string, :default => nil # host:port
+  config_param :standby_namenode, :string, default: nil # host:port
 
   desc 'Ignore errors on start up'
-  config_param :ignore_start_check_error, :bool, :default => false
-
-  include Fluent::Mixin::ConfigPlaceholders
+  config_param :ignore_start_check_error, :bool, default: false
 
   desc 'Output file path on HDFS'
   config_param :path, :string
   desc 'User name for pseudo authentication'
-  config_param :username, :string, :default => nil
+  config_param :username, :string, default: nil
 
   desc 'Store data over HttpFs instead of WebHDFS'
-  config_param :httpfs, :bool, :default => false
+  config_param :httpfs, :bool, default: false
 
   desc 'Number of seconds to wait for the connection to open'
-  config_param :open_timeout, :integer, :default => 30 # from ruby net/http default
+  config_param :open_timeout, :integer, default: 30 # from ruby net/http default
   desc 'Number of seconds to wait for one block to be read'
-  config_param :read_timeout, :integer, :default => 60 # from ruby net/http default
+  config_param :read_timeout, :integer, default: 60 # from ruby net/http default
 
   desc 'Retry automatically when known errors of HDFS are occurred'
-  config_param :retry_known_errors, :bool, :default => false
+  config_param :retry_known_errors, :bool, default: false
   desc 'Retry interval'
-  config_param :retry_interval, :integer, :default => nil
+  config_param :retry_interval, :integer, default: nil
   desc 'The number of retries'
-  config_param :retry_times, :integer, :default => nil
+  config_param :retry_times, :integer, default: nil
 
   # how many times of write failure before switch to standby namenode
   # by default it's 11 times that costs 1023 seconds inside fluentd,
   # which is considered enough to exclude the scenes that caused by temporary network fail or single datanode fail
   desc 'How many times of write failure before switch to standby namenode'
-  config_param :failures_before_use_standby, :integer, :default => 11
-
-  include Fluent::Mixin::PlainTextFormatter
+  config_param :failures_before_use_standby, :integer, default: 11
 
-  config_param :default_tag, :string, :default => 'tag_missing'
+  config_param :end_with_newline, :bool, default: true
 
   desc 'Append data or not'
-  config_param :append, :bool, :default => true
+  config_param :append, :bool, default: true
 
   desc 'Use SSL or not'
-  config_param :ssl, :bool, :default => false
+  config_param :ssl, :bool, default: false
   desc 'OpenSSL certificate authority file'
-  config_param :ssl_ca_file, :string, :default => nil
+  config_param :ssl_ca_file, :string, default: nil
   desc 'OpenSSL verify mode (none,peer)'
-  config_param :ssl_verify_mode, :default => nil do |val|
-    case val
-    when 'none'
-      :none
-    when 'peer'
-      :peer
-    else
-      raise Fluent::ConfigError, "unexpected parameter on ssl_verify_mode: #{val}"
-    end
-  end
+  config_param :ssl_verify_mode, :enum, list: [:none, :peer], default: :none
 
   desc 'Use kerberos authentication or not'
-  config_param :kerberos, :bool, :default => false
+  config_param :kerberos, :bool, default: false
 
-  SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'snappy', 'lzo_command', 'text']
+  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :text]
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
-  config_param :compress, :default => nil do |val|
-    unless SUPPORTED_COMPRESS.include? val
-      raise Fluent::ConfigError, "unsupported compress: #{val}"
-    end
-    val
-  end
+  config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
+
+  config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
+  config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
+  config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
+  config_param :suppress_log_broken_string, :bool, default: false, deprecated: "use @log_level for plugin to suppress such info logs"
 
   CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
 
-  attr_reader :compressor
+  config_section :buffer do
+    config_set_default :chunk_keys, ["time"]
+  end
+
+  config_section :format do
+    config_set_default :@type, 'out_file'
+    config_set_default :localtime, false # default timezone is UTC
+  end
+
+  attr_reader :formatter, :compressor
 
   def initialize
     super
-    require 'net/http'
-    require 'time'
-    require 'webhdfs'
-
     @compressor = nil
-  end
-
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
+    @standby_namenode_host = nil
+    @output_include_tag = @output_include_time = nil # TODO: deprecated
+    @header_separator = @field_separator = nil # TODO: deprecated
   end
 
   def configure(conf)
-    if conf['path']
-      if conf['path'].index('%S')
-        conf['time_slice_format'] = '%Y%m%d%H%M%S'
-      elsif conf['path'].index('%M')
-        conf['time_slice_format'] = '%Y%m%d%H%M'
-      elsif conf['path'].index('%H')
-        conf['time_slice_format'] = '%Y%m%d%H'
-      end
+    compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
+
+    timekey = case conf["path"]
+              when /%S/ then 1
+              when /%M/ then 60
+              when /%H/ then 3600
+              else 86400
+              end
+    if conf.elements(name: "buffer").empty?
+      e = Fluent::Config::Element.new("buffer", "time", {}, [])
+      conf.elements << e
     end
+    buffer_config = conf.elements(name: "buffer").first
+    buffer_config["timekey"] = timekey unless buffer_config["timekey"]
+
+    compat_parameters_convert_plaintextformatter(conf)
 
     super
 
-    begin
-      @compressor = COMPRESSOR_REGISTRY.lookup(@compress || 'text').new
-    rescue Fluent::ConfigError
-      raise
-    rescue
-      $log.warn "#{@comress} not found. Use 'text' instead"
-      @compressor = COMPRESSOR_REGISTRY.lookup('text').new
+    @formatter = formatter_create
+
+    if @using_formatter_config
+      @null_value = nil
+    else
+      @formatter.delimiter = "\x01" if @formatter.respond_to?(:delimiter) && @formatter.delimiter == 'SOH'
+      @null_value ||= 'NULL'
     end
 
+    if @default_tag.nil? && !@using_formatter_config && @output_include_tag
+      @default_tag = "tag_missing"
+    end
+    if @remove_prefix
+      @remove_prefix_actual = @remove_prefix + "."
+      @remove_prefix_actual_length = @remove_prefix_actual.length
+    end
+
+    verify_config_placeholders_in_path!(conf)
+    @replace_random_uuid = @path.include?('%{uuid}') || @path.include?('%{uuid_flush}')
+    if @replace_random_uuid
+      # to check SecureRandom.uuid is available or not (NotImplementedError raised in such environment)
+      begin
+        SecureRandom.uuid
+      rescue
+        raise Fluent::ConfigError, "uuid feature (SecureRandom) is unavailable in this environment"
+      end
+    end
+
+    @compressor = COMPRESSOR_REGISTRY.lookup(@compress.to_s).new
+
     if @host
       @namenode_host = @host
       @namenode_port = @port
@@ -167,7 +178,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
       @client_standby = nil
     end
 
-    if not @append
+    unless @append
       if @path.index(CHUNK_ID_PLACE_HOLDER).nil?
         raise Fluent::ConfigError, "path must contain ${chunk_id}, which is the placeholder for chunk_id, when append is set to false."
       end
@@ -230,14 +241,6 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
-  def shutdown
-    super
-  end
-
-  def path_format(chunk_key)
-    Time.strptime(chunk_key, @time_slice_format).strftime(@path)
-  end
-
   def is_standby_exception(e)
     e.is_a?(WebHDFS::IOError) && e.message.match(/org\.apache\.hadoop\.ipc\.StandbyException/)
   end
@@ -249,12 +252,6 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
-  def chunk_unique_id_to_str(unique_id)
-    unique_id.unpack('C*').map{|x| x.to_s(16).rjust(2,'0')}.join('')
-  end
-
-  # TODO check conflictions
-
   def send_data(path, data)
     if @append
       begin
@@ -267,13 +264,52 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
+  HOSTNAME_PLACEHOLDERS_DEPRECATED = ['${hostname}', '%{hostname}', '__HOSTNAME__']
+  UUID_RANDOM_PLACEHOLDERS_DEPRECATED = ['${uuid}', '${uuid:random}', '__UUID__', '__UUID_RANDOM__']
+  UUID_OTHER_PLACEHOLDERS_OBSOLETED = ['${uuid:hostname}', '%{uuid:hostname}', '__UUID_HOSTNAME__', '${uuid:timestamp}', '%{uuid:timestamp}', '__UUID_TIMESTAMP__']
+
+  def verify_config_placeholders_in_path!(conf)
+    return unless conf.has_key?('path')
+
+    path = conf['path']
+
+    # check @path for ${hostname}, %{hostname} and __HOSTNAME__ to warn to use #{Socket.gethostbyname}
+    if HOSTNAME_PLACEHOLDERS_DEPRECATED.any?{|ph| path.include?(ph) }
+      log.warn "hostname placeholder is now deprecated. use '\#\{Socket.gethostname\}' instead."
+      hostname = conf['hostname'] || Socket.gethostname
+      HOSTNAME_PLACEHOLDERS_DEPRECATED.each do |ph|
+        path.gsub!(ph, hostname)
+      end
+    end
+
+    if UUID_RANDOM_PLACEHOLDERS_DEPRECATED.any?{|ph| path.include?(ph) }
+      log.warn "random uuid placeholders are now deprecated. use %{uuid} (or %{uuid_flush}) instead."
+      UUID_RANDOM_PLACEHOLDERS_DEPRECATED.each do |ph|
+        path.gsub!(ph, '%{uuid}')
+      end
+    end
+
+    if UUID_OTHER_PLACEHOLDERS_OBSOLETED.any?{|ph| path.include?(ph) }
+      UUID_OTHER_PLACEHOLDERS_OBSOLETED.each do |ph|
+        if path.include?(ph)
+          log.error "configuration placeholder #{ph} is now unsupported by webhdfs output plugin."
+        end
+      end
+      raise ConfigError, "there are unsupported placeholders in path."
+    end
+  end
+
   def generate_path(chunk)
     hdfs_path = if @append
-                  path_format(chunk.key)
+                  extract_placeholders(@path, chunk.metadata)
                 else
-                  path_format(chunk.key).gsub(CHUNK_ID_PLACE_HOLDER, chunk_unique_id_to_str(chunk.unique_id))
+                  extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id(chunk.unique_id))
                 end
     hdfs_path = "#{hdfs_path}#{@compressor.ext}"
+    if @replace_random_uuid
+      uuid_random = SecureRandom.uuid
+      hdfs_path.gsub!('%{uuid}', uuid_random).gsub!('%{uuid_flush}', uuid_random)
+    end
     hdfs_path
   end
 
@@ -288,6 +324,48 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
+  def format(tag, time, record)
+    if @remove_prefix # TODO: remove when it's obsoleted
+      if tag.start_with?(@remove_prefix_actual)
+        if tag.length > @remove_prefix_actual_length
+          tag = tag[@remove_prefix_actual_length..-1]
+        else
+          tag = @default_tag
+        end
+      elsif tag.start_with?(@remove_prefix)
+        if tag == @remove_prefix
+          tag = @default_tag
+        else
+          tag = tag.sub(@remove_prefix, '')
+        end
+      end
+    end
+
+    if @null_value # TODO: remove when it's obsoleted
+      check_keys = (record.keys + @null_convert_keys).uniq
+      check_keys.each do |key|
+        record[key] = @null_value if record[key].nil?
+      end
+    end
+
+    if @using_formatter_config
+      record = inject_values_to_record(tag, time, record)
+      line = @formatter.format(tag, time, record)
+    else # TODO: remove when it's obsoleted
+      time_str = @output_include_time ? @time_formatter.call(time) + @header_separator : ''
+      tag_str = @output_include_tag ? tag + @header_separator : ''
+      record_str = @formatter.format(tag, time, record)
+      line = time_str + tag_str + record_str
+    end
+    line << "\n" if @end_with_newline && !line.end_with?("\n")
+    line
+  rescue => e # remove this clause when @suppress_log_broken_string is obsoleted
+    unless @suppress_log_broken_string
+      log.info "unexpected error while formatting events, ignored", tag: tag, record: record, error: e
+    end
+    ''
+  end
+
   def write(chunk)
     hdfs_path = generate_path(chunk)
 
@@ -318,6 +396,72 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     hdfs_path
   end
 
+  def compat_parameters_convert_plaintextformatter(conf)
+    if !conf.elements('format').empty? || !conf['output_data_type']
+      @using_formatter_config = true
+      @null_convert_keys = []
+      return
+    end
+
+    log.warn "webhdfs output plugin is working with old configuration parameters. use <inject>/<format> sections instead for further releases."
+    @using_formatter_config = false
+    @null_convert_keys = []
+
+    @header_separator = case conf['field_separator']
+                        when nil     then "\t"
+                        when 'SPACE' then ' '
+                        when 'TAB'   then "\t"
+                        when 'COMMA' then ','
+                        when 'SOH'   then "\x01"
+                        else conf['field_separator']
+                        end
+
+    format_section = Fluent::Config::Element.new('format', '', {}, [])
+    case conf['output_data_type']
+    when '', 'json' # blank value is for compatibility reason (especially in testing)
+      format_section['@type'] = 'json'
+    when 'ltsv'
+      format_section['@type'] = 'ltsv'
+    else
+      unless conf['output_data_type'].start_with?('attr:')
+        raise Fluent::ConfigError, "output_data_type is invalid: #{conf['output_data_type']}"
+      end
+      format_section['@format'] = 'tsv'
+      keys_part = conf['output_data_type'].sub(/^attr:/, '')
+      @null_convert_keys = keys_part.split(',')
+      format_section['keys'] = keys_part
+      format_section['delimiter'] = case conf['field_separator']
+                                    when nil then '\t'
+                                    when 'SPACE' then ' '
+                                    when 'TAB'   then '\t'
+                                    when 'COMMA' then ','
+                                    when 'SOH'   then 'SOH' # fixed later
+                                    else conf['field_separator']
+                                    end
+    end
+
+    conf.elements << format_section
+
+    @output_include_time = conf.has_key?('output_include_time') ? Fluent::Config.bool_value(conf['output_include_time']) : true
+    @output_include_tag = conf.has_key?('output_include_tag') ? Fluent::Config.bool_value(conf['output_include_tag']) : true
+
+    if @output_include_time
+      # default timezone is UTC
+      using_localtime = if !conf.has_key?('utc') && !conf.has_key?('localtime')
+                          false
+                        elsif conf.has_key?('localtime') && conf.has_key?('utc')
+                          raise Fluent::ConfigError, "specify either 'localtime' or 'utc'"
+                        elsif conf.has_key?('localtime')
+                          Fluent::Config.bool_value('localtime')
+                        else
+                          Fluent::Config.bool_value('utc')
+                        end
+      @time_formatter = Fluent::TimeFormatter.new(conf['time_format'], using_localtime)
+    else
+      @time_formatter = nil
+    end
+  end
+
   class Compressor
     include Fluent::Configurable
 

data/lib/fluent/plugin/webhdfs_compressor_bzip2.rb

@@ -1,5 +1,5 @@
-module Fluent
-  class WebHDFSOutput < Fluent::TimeSlicedOutput
+module Fluent::Plugin
+  class WebHDFSOutput < Output
     class Bzip2Compressor < Compressor
       WebHDFSOutput.register_compressor('bzip2', self)
 

data/lib/fluent/plugin/webhdfs_compressor_gzip.rb

@@ -1,5 +1,5 @@
-module Fluent
-  class WebHDFSOutput < Fluent::TimeSlicedOutput
+module Fluent::Plugin
+  class WebHDFSOutput < Output
     class GzipCompressor < Compressor
       WebHDFSOutput.register_compressor('gzip', self)
 

data/lib/fluent/plugin/webhdfs_compressor_lzo_command.rb

@@ -1,9 +1,9 @@
-module Fluent
-  class WebHDFSOutput
+module Fluent::Plugin
+  class WebHDFSOutput < Output
     class LZOCommandCompressor < Compressor
       WebHDFSOutput.register_compressor('lzo_command', self)
 
-      config_param :command_parameter, :string, :default => '-qf1'
+      config_param :command_parameter, :string, default: '-qf1'
 
       def configure(conf)
         super

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -1,5 +1,5 @@
-module Fluent
-  class WebHDFSOutput < Fluent::TimeSlicedOutput
+module Fluent::Plugin
+  class WebHDFSOutput < Output
     class SnappyCompressor < Compressor
       WebHDFSOutput.register_compressor('snappy', self)
 

data/lib/fluent/plugin/webhdfs_compressor_text.rb

@@ -1,5 +1,5 @@
-module Fluent
-  class WebHDFSOutput < Fluent::TimeSlicedOutput
+module Fluent::Plugin
+  class WebHDFSOutput < Output
     class TextCompressor < Compressor
       WebHDFSOutput.register_compressor('text', self)
 

data/test/helper.rb

@@ -8,10 +8,13 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'test/unit'
+require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|
@@ -22,6 +25,8 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
+include Fluent::Test::Helpers
+
 require 'fluent/plugin/out_webhdfs'
 
 class Test::Unit::TestCase

data/test/plugin/test_compressor.rb

@@ -16,11 +16,11 @@ class CompressorTest < Test::Unit::TestCase
     def setup
       omit unless Object.const_defined?(:Snappy)
       Fluent::Test.setup
-      @compressor = Fluent::WebHDFSOutput::SnappyCompressor.new
+      @compressor = Fluent::Plugin::WebHDFSOutput::SnappyCompressor.new
     end
 
-    def create_driver(conf=CONFIG,tag='test')
-      Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
+    def create_driver(conf = CONFIG)
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
     end
 
     def test_ext

data/test/plugin/test_out_webhdfs.rb

@@ -1,154 +1,243 @@
 require 'helper'
 
 class WebHDFSOutputTest < Test::Unit::TestCase
-  CONFIG = %[
-host namenode.local
-path /hdfs/path/file.%Y%m%d.log
-  ]
+  CONFIG_DEFAULT = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d.log"})
+
+  CONFIG_COMPAT = config_element(
+    "ROOT", "", {
+      "output_data_type" => "",
+      "host" => "namenode.local",
+      "path" => "/hdfs/path/file.%Y%m%d.log"
+    })
+
   def setup
     Fluent::Test.setup
   end
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
   end
 
-  class ConfigureTest < self
-    def test_default
-      d = create_driver
+  sub_test_case "default configuration" do
+    test 'configured with standard out_file format with specified hdfs info' do
+      d = create_driver(CONFIG_DEFAULT)
+      assert_true d.instance.instance_eval{ @using_formatter_config }
+
       assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 50070, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
-      assert_equal '%Y%m%d', d.instance.time_slice_format
       assert_equal false, d.instance.httpfs
       assert_nil d.instance.username
       assert_equal false, d.instance.ignore_start_check_error
 
-      assert_equal true, d.instance.output_include_time
-      assert_equal true, d.instance.output_include_tag
-      assert_equal 'json', d.instance.output_data_type
+      assert_equal 'Fluent::Plugin::OutFileFormatter', d.instance.formatter.class.to_s
+      assert_equal true, d.instance.end_with_newline
+
+      # deprecated params
+      assert_nil d.instance.instance_eval{ @output_include_time }
+      assert_nil d.instance.instance_eval{ @output_include_tag }
       assert_nil d.instance.remove_prefix
-      assert_equal 'TAB', d.instance.field_separator
-      assert_equal true, d.instance.add_newline
-      assert_equal 'tag_missing', d.instance.default_tag
+      assert_nil d.instance.instance_eval{ @header_separator }
+      assert_nil d.instance.default_tag
+    end
+  end
+
+  sub_test_case "flat configuration" do
+    def test_default_for_traditional_config
+      d = create_driver(CONFIG_COMPAT)
+      assert_false d.instance.instance_eval{ @using_formatter_config }
+
+      assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
+      assert_equal 50070, d.instance.instance_eval{ @namenode_port }
+      assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+      assert_equal false, d.instance.httpfs
+      assert_nil d.instance.username
+      assert_equal false, d.instance.ignore_start_check_error
+
+      assert_equal 'Fluent::Plugin::JSONFormatter', d.instance.formatter.class.to_s
+      assert_equal true, d.instance.end_with_newline
+
+      assert_equal true, d.instance.instance_eval{ @output_include_time }
+      assert_equal true, d.instance.instance_eval{ @output_include_tag }
+      assert_nil d.instance.instance_eval{ @remove_prefix }
+      assert_equal "\t", d.instance.instance_eval{ @header_separator }
+      assert_equal 'tag_missing', d.instance.instance_eval{ @default_tag }
     end
 
     def test_httpfs
-      d = create_driver %[
-namenode server.local:14000
-path /hdfs/path/file.%Y%m%d.%H%M.log
-httpfs yes
-username hdfs_user
-]
+      conf = config_element(
+        "ROOT", "", {
+          "namenode" => "server.local:14000",
+          "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+          "httpfs" => "yes",
+          "username" => "hdfs_user"
+        })
+      d = create_driver(conf)
+
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
       assert_equal true, d.instance.httpfs
       assert_equal 'hdfs_user', d.instance.username
     end
 
     def test_ssl
-      d = create_driver %[
-namenode server.local:14000
-path /hdfs/path/file.%Y%m%d.%H%M.log
-ssl true
-ssl_ca_file /path/to/ca_file.pem
-ssl_verify_mode peer
-kerberos true
-]
+      conf = config_element(
+        "ROOT", "", {
+          "namenode" => "server.local:14000",
+          "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+          "ssl" => true,
+          "ssl_ca_file" => "/path/to/ca_file.pem",
+          "ssl_verify_mode" => "peer",
+          "kerberos" => true
+        })
+      d = create_driver(conf)
+
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
       assert_equal true, d.instance.ssl
       assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
       assert_equal :peer, d.instance.ssl_verify_mode
       assert_equal true, d.instance.kerberos
     end
 
-    data(gzip: ['gzip', Fluent::WebHDFSOutput::GzipCompressor],
-         bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor],
-         snappy: ['snappy', Fluent::WebHDFSOutput::SnappyCompressor],
-         lzo: ['lzo_command', Fluent::WebHDFSOutput::LZOCommandCompressor])
+    data(gzip: [:gzip, Fluent::Plugin::WebHDFSOutput::GzipCompressor],
+         bzip2: [:bzip2, Fluent::Plugin::WebHDFSOutput::Bzip2Compressor],
+         snappy: [:snappy, Fluent::Plugin::WebHDFSOutput::SnappyCompressor],
+         lzo: [:lzo_command, Fluent::Plugin::WebHDFSOutput::LZOCommandCompressor])
     def test_compress(data)
       compress_type, compressor_class = data
       begin
-        d = create_driver %[
-namenode server.local:14000
-path /hdfs/path/file.%Y%m%d.%H%M.log
-compress #{compress_type}
-        ]
+        conf = config_element(
+          "ROOT", "", {
+            "namenode" => "server.local:14000",
+            "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+            "compress" => compress_type
+          })
+        d = create_driver(conf)
       rescue Fluent::ConfigError => ex
         omit ex.message
       end
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
       assert_equal compress_type, d.instance.compress
       assert_equal compressor_class, d.instance.compressor.class
     end
 
-    def test_placeholders
-      d = create_driver %[
-hostname testing.node.local
-namenode server.local:50070
-path /hdfs/${hostname}/file.%Y%m%d%H.log
-]
+    def test_placeholders_old_style
+      conf = config_element(
+        "ROOT", "", {
+          "hostname" => "testing.node.local",
+          "namenode" => "server.local:50070",
+          "path" => "/hdfs/${hostname}/file.%Y%m%d%H.log"
+        })
+      d = create_driver(conf)
       assert_equal '/hdfs/testing.node.local/file.%Y%m%d%H.log', d.instance.path
     end
 
-    class PathFormatTest < self
-      def test_default
-        d = create_driver
-        assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
-        assert_equal '%Y%m%d', d.instance.time_slice_format
-        assert_equal '/hdfs/path/file.20120718.log', d.instance.path_format('20120718')
-      end
+    data("%Y%m%d" => ["/hdfs/path/file.%Y%m%d.log", "/hdfs/path/file.20120718.log"],
+         "%Y%m%d.%H%M" => ["/hdfs/path/file.%Y%m%d.%H%M.log", "/hdfs/path/file.20120718.1503.log"])
+    test "generate_path" do |(path, expected)|
+      conf = config_element(
+        "ROOT", "", {
+          "namenode" => "server.local:14000",
+          "path" => path
+        })
+      d = create_driver(conf)
+      formatter = Fluent::Timezone.formatter("+0900", path)
+      mock(Fluent::Timezone).formatter(Time.now.strftime("%z"), path) { formatter }
+      time = event_time("2012-07-18 15:03:00 +0900")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal expected, d.instance.generate_path(chunk)
+    end
 
-      def test_time_slice_format
-        d = create_driver %[
-namenode server.local:14000
-path /hdfs/path/file.%Y%m%d.%H%M.log
-]
-        assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-        assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
-        assert_equal '/hdfs/path/file.20120718.1503.log', d.instance.path_format('201207181503')
+    data(path: { "append" => false },
+         ssl: { "ssl" => true, "ssl_verify_mode" => "invalid" },
+         compress: { "compress" => "invalid" })
+    test "invalid" do |attr|
+      conf = config_element(
+        "ROOT", "", {
+          "namenode" => "server.local:14000",
+          "path" => "/hdfs/path/file.%Y%m%d.%H%M.log"
+        })
+      conf += config_element("", "", attr)
+      assert_raise Fluent::ConfigError do
+        create_driver(conf)
       end
     end
+  end
 
-    class InvalidTest < self
-      def test_path
-        assert_raise Fluent::ConfigError do
-          d = create_driver %[
-            namenode server.local:14000
-            path /hdfs/path/file.%Y%m%d.%H%M.log
-            append false
-          ]
-        end
-      end
+  sub_test_case "sub section configuration" do
+    def test_time_key
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d.log"
+        }, [
+          config_element(
+            "buffer", "time", {
+              "timekey" => 1
+            })
+        ]
+      )
+      d = create_driver(conf)
+      time = event_time("2012-07-18 15:03:00 +0900")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal 1, d.instance.buffer_config.timekey
+      assert_equal "/hdfs/path/file.20120718.log", d.instance.generate_path(chunk)
+    end
+  end
 
-      def test_ssl
-        assert_raise Fluent::ConfigError do
-          create_driver %[
-            namenode server.local:14000
-            path /hdfs/path/file.%Y%m%d.%H%M.log
-            ssl true
-            ssl_verify_mode invalid
-          ]
-        end
-      end
+  sub_test_case "using format subsection" do
+    test "blank format means default format 'out_file' with UTC timezone" do
+      format_section = config_element("format", "", {}, [])
+      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [format_section])
+      d = create_driver(conf)
+      time = event_time("2017-01-24 13:10:30 -0700")
+      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
+      assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
+    end
 
-      def test_invalid_compress
-        assert_raise Fluent::ConfigError do
-          create_driver %[
-            namenode server.local:14000
-            path /hdfs/path/file.%Y%m%d.%H%M.log
-            compress invalid
-          ]
-        end
-      end
+    test "specifying timezone works well in format section" do
+      format_section = config_element("format", "", {"timezone" => "+0100"}, [])
+      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [format_section])
+      d = create_driver(conf)
+      time = event_time("2017-01-24 13:10:30 -0700")
+      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
+      assert_equal "2017-01-24T21:10:30+01:00\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
+    end
+
+    test "specifying formatter type LTSV for records, without tag and timezone" do
+      format_section = config_element("format", "", {"@type" => "ltsv"}, [])
+      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [format_section])
+      d = create_driver(conf)
+      time = event_time("2017-01-24 13:10:30 -0700")
+      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
+      assert_equal "message:yay\tname:tagomoris\n", line
+    end
+
+    test "specifying formatter type LTSV for records, with inject section to insert tag and time" do
+      inject_section = config_element("inject", "", {"tag_key" => "tag", "time_key" => "time", "time_type" => "string", "localtime" => "false"})
+      format_section = config_element("format", "", {"@type" => "ltsv"}, [])
+      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log"}, [inject_section, format_section])
+      d = create_driver(conf)
+      time = event_time("2017-01-24 13:10:30 -0700")
+      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
+      assert_equal "message:yay\tname:tagomoris\ttag:test.now\ttime:2017-01-24T20:10:30Z\n", line
+    end
+  end
+
+  sub_test_case "using older configuration" do
+    test "output_data_type json is same with out_file with UTC timezone" do
+      conf = config_element("match", "", {"host" => "namenode.local", "path" => "/hdfs/path/file.%Y%m%d%H.log", "output_data_type" => "json"}, [])
+      d = create_driver(conf)
+      time = event_time("2017-01-24 13:10:30 -0700")
+      line = d.instance.format("test.now", time, {"message" => "yay", "name" => "tagomoris"})
+      assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0rc1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-23 00:00:00.000000000 Z
+date: 2017-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: appraisal
+  name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,67 +53,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: snappy
+  name: appraisal
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.13
-- !ruby/object:Gem::Dependency
-  name: fluentd
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.59
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.14.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.59
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluent-mixin-plaintextformatter
+  name: snappy
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
-  type: :runtime
+        version: 0.0.13
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.0.13
 - !ruby/object:Gem::Dependency
-  name: fluent-mixin-config-placeholders
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.14.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.14.4
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -183,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.8