fluent-plugin-webhdfs 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ebfc106d895f4cb137cf632a3680312d6e6ed84
-  data.tar.gz: 84b238ea1329df8a871555c9aae43f0dc4659141
+  metadata.gz: 8da92c765c784f8e14759a04362206c34e30dc10
+  data.tar.gz: ae75ac7168ff93fd11000036d39fcdb0dd088186
 SHA512:
-  metadata.gz: a4a83f5591129c2e8649bf39d5a8ea1368f2389729a4362efb14ebefb6afebc9f1a3e89f8d90c0c5cb59e6238f9c2b1cde0727023d2c6e16967dfe104b34235c
-  data.tar.gz: de25df285af31e3aec52a2fcd0b04c55085a5a03dcdeaf0ba8862635e0b01f7a05b589f8364fa0842b44b9f7d48eaa90f017585e72313e86b58bbc52e5c384fe
+  metadata.gz: cd75219dc52eb63ded9f03673b4843ae55cc7bd4da179aa15d29639077287ee128fb1be24868f253218b43dd073412a2a7517272db3ad253a7f896fad470d754
+  data.tar.gz: e72169a9a84418fc776e43311eb842ed1bb6b58b83dcc9fab5b8a51cf75dcacd04fca4f9654706204bd8f62f96420e9fa20116febddd6e6168ced4fc5dce3a6f

data/README.md

@@ -10,6 +10,13 @@
 
 Paths on HDFS can be generated from event timestamp, tag or any other fields in records.
 
+## Requirements
+
+| fluent-plugin-webhdfs | fluentd    | ruby   |
+|-----------------------|------------|--------|
+| >= 1.0.0              | >= v0.14.4 | >= 2.1 |
+| <  1.0.0              | <  v0.14.0 | >= 1.9 |
+
 ### Older versions
 
 The versions of `0.x.x` of this plugin are for older version of Fluentd (v0.12.x). Old style configuration parameters (using `output_data_type`, `output_include_*` or others) are still supported, but are deprecated.

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.1.0"
+  gem.version       = "1.1.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}

data/lib/fluent/plugin/out_webhdfs.rb

@@ -312,7 +312,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     hdfs_path = "#{hdfs_path}#{@compressor.ext}"
     if @replace_random_uuid
       uuid_random = SecureRandom.uuid
-      hdfs_path.gsub!('%{uuid}', uuid_random).gsub!('%{uuid_flush}', uuid_random)
+      hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)
     end
     hdfs_path
   end

data/lib/fluent/plugin/webhdfs_compressor_gzip.rb

@@ -14,7 +14,7 @@ module Fluent::Plugin
       def compress(chunk, tmp)
         w = Zlib::GzipWriter.new(tmp)
         chunk.write_to(w)
-        w.close
+        w.finish
       end
     end
   end

data/test/plugin/test_gzip_compressor.rb

@@ -0,0 +1,57 @@
+require "helper"
+require "fluent/plugin/buf_memory"
+begin
+  require "zlib"
+rescue LoadError
+end
+
+class GzipCompressorTest < Test::Unit::TestCase
+  class Gzip < self
+
+    CONFIG = %[
+      host namenode.local
+      path /hdfs/path/file.%Y%m%d.log
+    ]
+
+    def setup
+      omit unless Object.const_defined?(:Zlib)
+      Fluent::Test.setup
+      @compressor = Fluent::Plugin::WebHDFSOutput::GzipCompressor.new
+    end
+
+    def create_driver(conf = CONFIG)
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
+    end
+
+    def test_ext
+      assert_equal(".gz", @compressor.ext)
+    end
+
+    def test_compress
+      d = create_driver
+      if d.instance.respond_to?(:buffer)
+        buffer = d.instance.buffer
+      else
+        buffer = d.instance.instance_variable_get(:@buffer)
+      end
+
+      if buffer.respond_to?(:generate_chunk)
+        chunk = buffer.generate_chunk("test")
+        chunk.concat("hello gzip\n" * 32 * 1024, 1)
+      else
+        chunk = buffer.new_chunk("test")
+        chunk << "hello gzip\n" * 32 * 1024
+      end
+
+      io = Tempfile.new("gzip-")
+      @compressor.compress(chunk, io)
+      assert !io.closed?
+      chunk_bytesize = chunk.respond_to?(:bytesize) ? chunk.bytesize : chunk.size
+      assert(chunk_bytesize > io.read.bytesize)
+      io.rewind
+      reader = Zlib::GzipReader.new(io)
+      assert_equal(chunk.read, reader.read)
+      io.close
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-31 00:00:00.000000000 Z
+date: 2017-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -145,6 +145,7 @@ files:
 - lib/fluent/plugin/webhdfs_compressor_text.rb
 - test/helper.rb
 - test/plugin/test_compressor.rb
+- test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses:
@@ -166,11 +167,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
 test_files:
 - test/helper.rb
 - test/plugin/test_compressor.rb
+- test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb