fluent-plugin-webhdfs 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8865ee69f790536d7ce119ead16abc7862efdf59
-  data.tar.gz: df3f7cc64b42d465733ca51815b9e066b9d4310c
+  metadata.gz: 7ebfc106d895f4cb137cf632a3680312d6e6ed84
+  data.tar.gz: 84b238ea1329df8a871555c9aae43f0dc4659141
 SHA512:
-  metadata.gz: 384e20026a6b64a91c3ca59058e6e4e33480dda26057165cf9464006bc7e0b6d61170d97878abfe5c8c29a23b263e7a6db770f4cfcf4170b5de57ee3f2337da6
-  data.tar.gz: 464ec8333c3c97002150cabbeed4f13077abfe473944f4a56fb764414f25e6e03e793942f11ca270695c22cc813866e63c802debfa6f967ac08f98e4637b9e5d
+  metadata.gz: a4a83f5591129c2e8649bf39d5a8ea1368f2389729a4362efb14ebefb6afebc9f1a3e89f8d90c0c5cb59e6238f9c2b1cde0727023d2c6e16967dfe104b34235c
+  data.tar.gz: de25df285af31e3aec52a2fcd0b04c55085a5a03dcdeaf0ba8862635e0b01f7a05b589f8364fa0842b44b9f7d48eaa90f017585e72313e86b58bbc52e5c384fe

data/.travis.yml

@@ -4,7 +4,8 @@ language: ruby
 rvm:
   - 2.1
   - 2.2
-  - 2.3.1
+  - 2.3.3
+  - 2.4.0
 
 branches:
   only:
@@ -22,4 +23,3 @@ script: bundle exec rake test
 
 gemfile:
   - Gemfile
-  - gemfiles/fluentd_v0.14.gemfile

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.0.0"
+  gem.version       = "1.1.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}

data/lib/fluent/plugin/out_webhdfs.rb

@@ -112,6 +112,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     buffer_config["timekey"] = timekey unless buffer_config["timekey"]
 
     compat_parameters_convert_plaintextformatter(conf)
+    verify_config_placeholders_in_path!(conf)
 
     super
 
@@ -132,7 +133,6 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       @remove_prefix_actual_length = @remove_prefix_actual.length
     end
 
-    verify_config_placeholders_in_path!(conf)
     @replace_random_uuid = @path.include?('%{uuid}') || @path.include?('%{uuid_flush}')
     if @replace_random_uuid
       # to check SecureRandom.uuid is available or not (NotImplementedError raised in such environment)
@@ -185,6 +185,10 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
   end
 
+  def multi_workers_ready?
+    true
+  end
+
   def prepare_client(host, port, username)
     client = WebHDFS::Client.new(host, port, username)
     if @httpfs
@@ -303,7 +307,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     hdfs_path = if @append
                   extract_placeholders(@path, chunk.metadata)
                 else
-                  extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id(chunk.unique_id))
+                  extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id))
                 end
     hdfs_path = "#{hdfs_path}#{@compressor.ext}"
     if @replace_random_uuid

data/test/helper.rb

@@ -26,6 +26,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 include Fluent::Test::Helpers
+include Fluent::UniqueId::Mixin
 
 require 'fluent/plugin/out_webhdfs'
 

data/test/plugin/test_out_webhdfs.rb

@@ -154,6 +154,20 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal expected, d.instance.generate_path(chunk)
     end
 
+    data(path: "/hdfs/path/file.${chunk_id}.log")
+    test "generate_path without append" do |path|
+      conf = config_element(
+        "ROOT", "", {
+          "namenode" => "server.local:14000",
+          "path" => path,
+          "append" => false
+        })
+      d = create_driver(conf)
+      metadata = d.instance.metadata("test", nil, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.#{dump_unique_id_hex(chunk.unique_id)}.log", d.instance.generate_path(chunk)
+    end
+
     data(path: { "append" => false },
          ssl: { "ssl" => true, "ssl_verify_mode" => "invalid" },
          compress: { "compress" => "invalid" })

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-26 00:00:00.000000000 Z
+date: 2017-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -137,8 +137,6 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-webhdfs.gemspec
-- gemfiles/fluentd_v0.12.gemfile
-- gemfiles/fluentd_v0.14.gemfile
 - lib/fluent/plugin/out_webhdfs.rb
 - lib/fluent/plugin/webhdfs_compressor_bzip2.rb
 - lib/fluent/plugin/webhdfs_compressor_gzip.rb

data/gemfiles/fluentd_v0.12.gemfile

@@ -1,7 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "fluentd", "~>0.12.0"
-
-gemspec :path => "../"

data/gemfiles/fluentd_v0.14.gemfile

@@ -1,7 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "fluentd", "~>0.14.0"
-
-gemspec :path => "../"