fluent-plugin-webhdfs 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6318fcc2f54eb64be7d99978a04b1cad30e740734ed0dafdf1441cf65988f1ae
-  data.tar.gz: 1d49b9c8b009a9a6078c4f9e9fe08b79832510cc780e50344ea72e8493c8e5c1
+  metadata.gz: bc42357da759e1c34ec12b3994bdd96b9f56cc1b093bb890f0ec4bccf929362d
+  data.tar.gz: e63cb6a5df15e5cf2fe8228d9e0e21ff5adf6db9cc5c4e11138aaac77429dc85
 SHA512:
-  metadata.gz: 16f6c451fe30d09b33255dcac7ed8714beb0c157fee3bdce8e041b1de9c0aa032910b8cf21ac1effd8dddbea4240a44c5e68e3a83275e386fd881dbb22c75b1d
-  data.tar.gz: c59533a82d5c46e9ea178402ede6852aa4d8fa7375981433b7f5b16311a55bb4e58234a0e28b657e318b690946c28622bfbda5da6b4318549342a394d6b8c15a
+  metadata.gz: e613ca241b2624ac77c1b1651de28aad3b4e7060086067d1eefc4874241e0a60437bddf61f3775c834a94fdaf1cd374fad1cb5b60e16909db49cf9dc7663770b
+  data.tar.gz: 7eb4b39ab4763f661e1e213d736eb3544d88e1a73b24a0b0d59b6a715fa3fdbeceb62f5a47d14be6d06126747d4751f0b73fa7267591c6d2713d2dac35901f5e

data/.github/workflows/linux.yml

@@ -0,0 +1,35 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: sudo apt-get install libsnappy-dev libzstd-dev
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.travis.yml

@@ -2,10 +2,11 @@ sudo: false
 language: ruby
 
 rvm:
-  - 2.1
-  - 2.2
-  - 2.3.3
-  - 2.4.0
+  - 2.4
+  - 2.5
+  - 2.6
+  - 2.7
+  - ruby-head
 
 branches:
   only:
@@ -23,3 +24,7 @@ script: bundle exec rake test
 
 gemfile:
   - Gemfile
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/README.md

@@ -146,6 +146,7 @@ With kerberos authentication:
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
       kerberos true
       kerberos_keytab /path/to/keytab # if needed
+      renew_kerberos_delegation_token true # if needed
     </match>
 
 NOTE: You need to install `gssapi` gem for kerberos. See https://github.com/kzk/webhdfs#for-kerberos-authentication
@@ -157,18 +158,44 @@ If you want to compress data before storing it:
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H
-      compress gzip  # or 'bzip2', 'snappy', 'lzo_command', 'zstd'
+      compress gzip  # or 'bzip2', 'snappy', 'hadoop_snappy', 'lzo_command', 'zstd'
     </match>
 
-Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`, `.zst`).
+Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `.sz`, `.snappy`, `.lzo`, `.zst`).
 Note that you have to install additional gem for several compress algorithms:
 
 - snappy: install snappy gem
+- hadoop_snappy: install snappy gem
 - bzip2: install bzip2-ffi gem
 - zstd: install zstandard gem
 
 Note that zstd will require installation of the libzstd native library. See the [zstandard-ruby](https://github.com/msievers/zstandard-ruby#examples-for-installing-libzstd) repo for infomration on the required packages for your operating system.
 
+You can also specify compression block size (currently supported only for Snappy codecs):
+
+    <match access.**>
+      @type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H
+      compress hadoop_snappy
+      block_size 32768
+    </match>
+
+If you want to explicitly specify file extensions in HDFS (override default compressor extensions):
+
+    <match access.**>
+      @type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H
+      compress snappy
+      extension ".snappy"
+    </match>
+
+With this configuration paths in HDFS will be like `/path/on/hdfs/access.log.20201003_12.snappy`.
+This one may be useful when (for example) you need to use snappy codec but `.sz` files are not recognized as snappy files in HDFS.
+
 ### Namenode HA / Auto retry for WebHDFS known errors
 
 `fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:
@@ -228,7 +255,7 @@ For high load cluster nodes, you can specify timeouts for HTTP requests.
 
 ### For unstable Namenodes
 
-With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inacive NameNodes.
+With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inactive NameNodes.
 
 If you were using unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
 

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.3.0"
+  gem.version       = "1.5.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -23,5 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "bzip2-ffi"
   gem.add_development_dependency "zstandard"
   gem.add_runtime_dependency "fluentd", '>= 0.14.22'
-  gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
+  gem.add_runtime_dependency "webhdfs", '>= 0.10.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -66,11 +66,18 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   config_param :kerberos, :bool, default: false
   desc 'kerberos keytab file'
   config_param :kerberos_keytab, :string, default: nil
+  desc 'Use delegation token while upload webhdfs or not'
+  config_param :renew_kerberos_delegation_token, :bool, default: false
+  desc 'delegation token reuse timer (default 8h)'
+  config_param :renew_kerberos_delegation_token_interval, :time, default: 8 * 60 * 60
 
-  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :zstd, :text]
-  desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
+  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :hadoop_snappy, :lzo_command, :zstd, :text]
+  desc "Compression method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
 
+  desc 'HDFS file extensions (overrides default compressor extensions)'
+  config_param :extension, :string, default: nil
+
   config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
   config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
   config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
@@ -153,6 +160,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
 
     @compressor = COMPRESSOR_REGISTRY.lookup(@compress.to_s).new
+    @compressor.configure(conf)
 
     if @host
       @namenode_host = @host
@@ -180,6 +188,14 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       raise Fluent::ConfigError, "Path on hdfs MUST starts with '/', but '#{@path}'"
     end
 
+    @renew_kerberos_delegation_token_interval_hour = nil
+    if @renew_kerberos_delegation_token
+      unless @username
+        raise Fluent::ConfigError, "username is missing. If you want to reuse delegation token, follow with kerberos accounts"
+      end
+      @renew_kerberos_delegation_token_interval_hour = @renew_kerberos_delegation_token_interval / 60 / 60
+    end
+
     @client = prepare_client(@namenode_host, @namenode_port, @username)
     if @standby_namenode_host
       @client_standby = prepare_client(@standby_namenode_host, @standby_namenode_port, @username)
@@ -199,7 +215,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   end
 
   def prepare_client(host, port, username)
-    client = WebHDFS::Client.new(host, port, username)
+    client = WebHDFS::Client.new(host, port, username, nil, nil, nil, {}, @renew_kerberos_delegation_token_interval_hour)
     if @httpfs
       client.httpfs_mode = true
     end
@@ -267,17 +283,22 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   end
 
   def send_data(path, data)
-    if @append
-      begin
-        @client.append(path, data)
-      rescue WebHDFS::FileNotFoundError
-        @client.create(path, data)
-      end
+    return @client.create(path, data, {'overwrite' => 'true'}) unless @append
+
+    if path_exists?(path)
+      @client.append(path, data)
     else
-      @client.create(path, data, {'overwrite' => 'true'})
+      @client.create(path, data)
     end
   end
 
+  def path_exists?(path)
+    @client.stat(path)
+    true
+  rescue WebHDFS::FileNotFoundError
+    false
+  end
+
   HOSTNAME_PLACEHOLDERS_DEPRECATED = ['${hostname}', '%{hostname}', '__HOSTNAME__']
   UUID_RANDOM_PLACEHOLDERS_DEPRECATED = ['${uuid}', '${uuid:random}', '__UUID__', '__UUID_RANDOM__']
   UUID_OTHER_PLACEHOLDERS_OBSOLETED = ['${uuid:hostname}', '%{uuid:hostname}', '__UUID_HOSTNAME__', '${uuid:timestamp}', '%{uuid:timestamp}', '__UUID_TIMESTAMP__']
@@ -319,7 +340,8 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
                 else
                   extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk)
                 end
-    hdfs_path = "#{hdfs_path}#{@compressor.ext}"
+    hdfs_ext = @extension || @compressor.ext
+    hdfs_path = "#{hdfs_path}#{hdfs_ext}"
     if @replace_random_uuid
       uuid_random = SecureRandom.uuid
       hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)
@@ -502,7 +524,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       begin
         Open3.capture3("#{command} -V")
       rescue Errno::ENOENT
-        raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+        raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
       end
     end
   end
@@ -518,5 +540,6 @@ require 'fluent/plugin/webhdfs_compressor_text'
 require 'fluent/plugin/webhdfs_compressor_gzip'
 require 'fluent/plugin/webhdfs_compressor_bzip2'
 require 'fluent/plugin/webhdfs_compressor_snappy'
+require 'fluent/plugin/webhdfs_compressor_hadoop_snappy'
 require 'fluent/plugin/webhdfs_compressor_lzo_command'
 require 'fluent/plugin/webhdfs_compressor_zstd'

data/lib/fluent/plugin/webhdfs_compressor_hadoop_snappy.rb

@@ -0,0 +1,32 @@
+module Fluent::Plugin
+  class WebHDFSOutput < Output
+    class HadoopSnappyCompressor < Compressor
+      WebHDFSOutput.register_compressor('hadoop_snappy', self)
+
+      DEFAULT_BLOCK_SIZE = 256 * 1024
+
+      desc 'Block size for compression algorithm'
+      config_param :block_size, :integer, default: DEFAULT_BLOCK_SIZE
+
+      def initialize(options = {})
+        super()
+        begin
+          require "snappy"
+        rescue LoadError
+          raise Fluent::ConfigError, "Install snappy before using snappy compressor"
+        end
+      end
+
+      def ext
+        ".snappy"
+      end
+
+      def compress(chunk, tmp)
+        Snappy::Hadoop::Writer.new(tmp, @block_size) do |w|
+          w << chunk.read
+          w.flush
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/webhdfs_compressor_lzo_command.rb

@@ -5,8 +5,8 @@ module Fluent::Plugin
 
       config_param :command_parameter, :string, default: '-qf1'
 
-      def configure(conf)
-        super
+      def initialize(options = {})
+        super()
         check_command('lzop', 'LZO')
       end
 

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -3,11 +3,17 @@ module Fluent::Plugin
     class SnappyCompressor < Compressor
       WebHDFSOutput.register_compressor('snappy', self)
 
+      DEFAULT_BLOCK_SIZE = 32 * 1024
+
+      desc 'Block size for compression algorithm'
+      config_param :block_size, :integer, default: DEFAULT_BLOCK_SIZE
+
       def initialize(options = {})
+        super()
         begin
           require "snappy"
         rescue LoadError
-          raise Fluent::ConfigError, "Install snappy before use snappy compressor"
+          raise Fluent::ConfigError, "Install snappy before using snappy compressor"
         end
       end
 
@@ -16,7 +22,7 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
-        Snappy::Writer.new(tmp) do |w|
+        Snappy::Writer.new(tmp, @block_size) do |w|
           w << chunk.read
           w.flush
         end

data/test/plugin/test_out_webhdfs.rb

@@ -107,6 +107,7 @@ class WebHDFSOutputTest < Test::Unit::TestCase
     data(gzip: [:gzip, Fluent::Plugin::WebHDFSOutput::GzipCompressor],
          bzip2: [:bzip2, Fluent::Plugin::WebHDFSOutput::Bzip2Compressor],
          snappy: [:snappy, Fluent::Plugin::WebHDFSOutput::SnappyCompressor],
+         hadoop_snappy: [:hadoop_snappy, Fluent::Plugin::WebHDFSOutput::HadoopSnappyCompressor],
          lzo: [:lzo_command, Fluent::Plugin::WebHDFSOutput::LZOCommandCompressor])
     def test_compress(data)
       compress_type, compressor_class = data
@@ -116,7 +117,13 @@ class WebHDFSOutputTest < Test::Unit::TestCase
             "namenode" => "server.local:14000",
             "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
             "compress" => compress_type
-          })
+          }, [
+            config_element("buffer", "tag, time", {
+                             "@type" => "memory",
+                             "timekey_zone" => "+0300",
+                             "timekey" => 60
+                           })
+          ])
         d = create_driver(conf)
       rescue Fluent::ConfigError => ex
         omit ex.message
@@ -126,6 +133,43 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
       assert_equal compress_type, d.instance.compress
       assert_equal compressor_class, d.instance.compressor.class
+
+      time = event_time("2020-10-03 15:07:00 +0300")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.20201003.1507.log#{d.instance.compressor.ext}", d.instance.generate_path(chunk)
+    end
+
+    def test_explicit_extensions
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d.log",
+          "compress" => "snappy",
+          "extension" => ".snappy"
+        })
+      d = create_driver(conf)
+      time = event_time("2020-10-07 15:15:00 +0300")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.20201007.log.snappy", d.instance.generate_path(chunk)
+    end
+
+    data(snappy: [:snappy, Fluent::Plugin::WebHDFSOutput::SnappyCompressor],
+         hadoop_snappy: [:hadoop_snappy, Fluent::Plugin::WebHDFSOutput::HadoopSnappyCompressor])
+    def test_compression_block_size(data)
+      compress_type, compressor_class = data
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d.log",
+          "compress" => compress_type,
+          "block_size" => 16384
+        })
+      d = create_driver(conf)
+
+      assert_equal compress_type, d.instance.compress
+      assert_equal 16384, d.instance.compressor.block_size
     end
 
     def test_placeholders_old_style
@@ -272,4 +316,99 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
     end
   end
+
+  sub_test_case "kerberos config" do
+    CONFIG_KERBEROS = config_element(
+      "ROOT", "", {
+        "namenode" => "server.local:14000",
+        "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+        "username" => "hdfs_user",
+        "kerberos" => true,
+        "kerberos_keytab" => "/path/to/kerberos.keytab",
+      })
+
+    test "renew_kerberos_delegation_token default" do
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, nil).once
+
+      d = create_driver(CONFIG_KERBEROS)
+
+      assert_equal(
+        {
+          kerberos: true,
+          renew_kerberos_delegation_token: false,
+          renew_kerberos_delegation_token_interval_hour: nil,
+        },
+        {
+          kerberos: d.instance.kerberos,
+          renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
+          renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+        })
+    end
+
+    test "default renew_kerberos_delegation_token_interval" do
+      expected_hour = 8
+
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour).once
+
+      d = create_driver(CONFIG_KERBEROS +
+                        config_element("", "", { "renew_kerberos_delegation_token" => true }))
+
+      assert_equal(
+        {
+          kerberos: true,
+          renew_kerberos_delegation_token: true,
+          renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
+          renew_kerberos_delegation_token_interval_hour: expected_hour,
+        },
+        {
+          kerberos: d.instance.kerberos,
+          renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
+          renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
+          renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+        })
+    end
+
+    test "renew_kerberos_delegation_token_interval" do
+      expected_hour = 10
+
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour).once
+
+      d = create_driver(
+        CONFIG_KERBEROS +
+        config_element(
+          "", "",
+          {
+            "renew_kerberos_delegation_token" => true,
+            "renew_kerberos_delegation_token_interval" => "#{expected_hour}h",
+          }))
+
+      assert_equal(
+        {
+          kerberos: true,
+          renew_kerberos_delegation_token: true,
+          renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
+          renew_kerberos_delegation_token_interval_hour: expected_hour,
+        },
+        {
+          kerberos: d.instance.kerberos,
+          renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
+          renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
+          renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+        })
+    end
+
+    test "username is required for renew_kerberos_delegation_token" do
+      conf = config_element(
+      "ROOT", "", {
+        "namenode" => "server.local:14000",
+        "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+        "kerberos" => true,
+        "renew_kerberos_delegation_token" => true,
+      })
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver(conf)
+      end
+    end
+  end
 end

data/test/plugin/{test_compressor.rb → test_snappy_compressors.rb}

@@ -5,7 +5,7 @@ begin
 rescue LoadError
 end
 
-class CompressorTest < Test::Unit::TestCase
+class SnappyCompressorsTest < Test::Unit::TestCase
   class Snappy < self
 
     CONFIG = %[
@@ -16,7 +16,17 @@ class CompressorTest < Test::Unit::TestCase
     def setup
       omit unless Object.const_defined?(:Snappy)
       Fluent::Test.setup
-      @compressor = Fluent::Plugin::WebHDFSOutput::SnappyCompressor.new
+
+      @compressors_size = 2
+      @compressors = [
+        Fluent::Plugin::WebHDFSOutput::SnappyCompressor.new,
+        Fluent::Plugin::WebHDFSOutput::HadoopSnappyCompressor.new
+      ]
+      @readers = [
+        ::Snappy::Reader,
+        ::Snappy::Hadoop::Reader
+      ]
+      @exts = [".sz", ".snappy"]
     end
 
     def create_driver(conf = CONFIG)
@@ -24,7 +34,9 @@ class CompressorTest < Test::Unit::TestCase
     end
 
     def test_ext
-      assert_equal(".sz", @compressor.ext)
+      for i in 0...@compressors_size do
+        assert_equal(@exts[i], @compressors[i].ext)
+      end
     end
 
     def test_compress
@@ -43,15 +55,17 @@ class CompressorTest < Test::Unit::TestCase
         chunk << "hello snappy\n" * 32 * 1024
       end
 
-      io = Tempfile.new("snappy-")
-      @compressor.compress(chunk, io)
-      io.open
-      chunk_bytesize = chunk.respond_to?(:bytesize) ? chunk.bytesize : chunk.size
-      assert(chunk_bytesize > io.read.bytesize)
-      io.rewind
-      reader = ::Snappy::Reader.new(io)
-      assert_equal(chunk.read, reader.read)
-      io.close
+      for i in 0...@compressors_size do
+        io = Tempfile.new("snappy-")
+        @compressors[i].compress(chunk, io)
+        io.open
+        chunk_bytesize = chunk.respond_to?(:bytesize) ? chunk.bytesize : chunk.size
+        assert(chunk_bytesize > io.read.bytesize)
+        io.rewind
+        reader = @readers[i].new(io)
+        assert_equal(chunk.read, reader.read)
+        io.close
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2021-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.10.0
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -143,6 +143,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".travis.yml"
 - Appraisals
@@ -154,14 +155,15 @@ files:
 - lib/fluent/plugin/out_webhdfs.rb
 - lib/fluent/plugin/webhdfs_compressor_bzip2.rb
 - lib/fluent/plugin/webhdfs_compressor_gzip.rb
+- lib/fluent/plugin/webhdfs_compressor_hadoop_snappy.rb
 - lib/fluent/plugin/webhdfs_compressor_lzo_command.rb
 - lib/fluent/plugin/webhdfs_compressor_snappy.rb
 - lib/fluent/plugin/webhdfs_compressor_text.rb
 - lib/fluent/plugin/webhdfs_compressor_zstd.rb
 - test/helper.rb
-- test/plugin/test_compressor.rb
 - test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
+- test/plugin/test_snappy_compressors.rb
 - test/plugin/test_zstd_compressor.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses:
@@ -182,13 +184,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.5
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
 test_files:
 - test/helper.rb
-- test/plugin/test_compressor.rb
 - test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
+- test/plugin/test_snappy_compressors.rb
 - test/plugin/test_zstd_compressor.rb