fluent-plugin-webhdfs 1.2.5 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a0c203ea54e21d95aea500fbb547ea965d5a5c07c1b277b69df2bbc44fa50fe
-  data.tar.gz: 6c93680eeba4575e2ef817e58772d4eb64a2e205696ad87b4b6decc1ed300e90
+  metadata.gz: 6318fcc2f54eb64be7d99978a04b1cad30e740734ed0dafdf1441cf65988f1ae
+  data.tar.gz: 1d49b9c8b009a9a6078c4f9e9fe08b79832510cc780e50344ea72e8493c8e5c1
 SHA512:
-  metadata.gz: 91be9ef776972c9d149221b2de2b765890e43e6df1e2a2408876b353aae629552a64d8029cfda84d8c03510e77ac0e164c7a8ed379d8d878505764df6181ca7e
-  data.tar.gz: 4b82ec9889bbd1a4e4fdd03150d082477a71ac03bacf5a72510e0dd816f8cfdb7e16dc5dc40ed02fdd2118f3a6e35919cf7032b0c7194bbdf76965dfc61c587e
+  metadata.gz: 16f6c451fe30d09b33255dcac7ed8714beb0c157fee3bdce8e041b1de9c0aa032910b8cf21ac1effd8dddbea4240a44c5e68e3a83275e386fd881dbb22c75b1d
+  data.tar.gz: c59533a82d5c46e9ea178402ede6852aa4d8fa7375981433b7f5b16311a55bb4e58234a0e28b657e318b690946c28622bfbda5da6b4318549342a394d6b8c15a

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+vendor/

data/README.md

@@ -157,14 +157,17 @@ If you want to compress data before storing it:
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H
-      compress gzip  # or 'bzip2', 'snappy', 'lzo_command'
+      compress gzip  # or 'bzip2', 'snappy', 'lzo_command', 'zstd'
     </match>
 
-Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`).
+Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`, `.zst`).
 Note that you have to install additional gem for several compress algorithms:
 
 - snappy: install snappy gem
 - bzip2: install bzip2-ffi gem
+- zstd: install zstandard gem
+
+Note that zstd will require installation of the libzstd native library. See the [zstandard-ruby](https://github.com/msievers/zstandard-ruby#examples-for-installing-libzstd) repo for infomration on the required packages for your operating system.
 
 ### Namenode HA / Auto retry for WebHDFS known errors
 

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.2.5"
+  gem.version       = "1.3.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -21,6 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "appraisal"
   gem.add_development_dependency "snappy", '>= 0.0.13'
   gem.add_development_dependency "bzip2-ffi"
+  gem.add_development_dependency "zstandard"
   gem.add_runtime_dependency "fluentd", '>= 0.14.22'
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -67,7 +67,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   desc 'kerberos keytab file'
   config_param :kerberos_keytab, :string, default: nil
 
-  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :text]
+  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :zstd, :text]
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
 
@@ -519,3 +519,4 @@ require 'fluent/plugin/webhdfs_compressor_gzip'
 require 'fluent/plugin/webhdfs_compressor_bzip2'
 require 'fluent/plugin/webhdfs_compressor_snappy'
 require 'fluent/plugin/webhdfs_compressor_lzo_command'
+require 'fluent/plugin/webhdfs_compressor_zstd'

data/lib/fluent/plugin/webhdfs_compressor_zstd.rb

@@ -0,0 +1,24 @@
+module Fluent::Plugin
+  class WebHDFSOutput < Output
+    class ZstdCompressor < Compressor
+      WebHDFSOutput.register_compressor('zstd', self)
+
+      def initialize(options = {})
+        begin
+          require "zstandard"
+        rescue LoadError
+          raise Fluent::ConfigError, "Install zstandard gem before use of zstd compressor"
+        end
+      end
+
+      def ext
+        ".zst"
+      end
+
+      def compress(chunk, tmp)
+        tmp.binmode
+        tmp.write Zstandard.deflate(chunk.read)
+      end
+    end
+  end
+end

data/test/plugin/test_zstd_compressor.rb

@@ -0,0 +1,55 @@
+require "helper"
+require "fluent/plugin/buf_memory"
+require 'zstandard'
+
+class ZstdCompressorTest < Test::Unit::TestCase
+  class Zstd < self
+
+    CONFIG = %[
+      host namenode.local
+      path /hdfs/path/file.%Y%m%d.log
+    ]
+
+    def setup
+      omit unless Object.const_defined?(:Zstandard)
+      Fluent::Test.setup
+      @compressor = Fluent::Plugin::WebHDFSOutput::ZstdCompressor.new
+    end
+
+    def create_driver(conf = CONFIG)
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
+    end
+
+    def test_ext
+      assert_equal(".zst", @compressor.ext)
+    end
+
+    def test_compress
+      d = create_driver
+      if d.instance.respond_to?(:buffer)
+        buffer = d.instance.buffer
+      else
+        buffer = d.instance.instance_variable_get(:@buffer)
+      end
+
+      if buffer.respond_to?(:generate_chunk)
+        chunk = buffer.generate_chunk("test")
+        chunk.concat("hello gzip\n" * 32 * 1024, 1)
+      else
+        chunk = buffer.new_chunk("test")
+        chunk << "hello gzip\n" * 32 * 1024
+      end
+
+      io = Tempfile.new("zstd-")
+      @compressor.compress(chunk, io)
+      assert !io.closed?
+      chunk_bytesize = chunk.respond_to?(:bytesize) ? chunk.bytesize : chunk.size
+      assert(chunk_bytesize > io.read.bytesize)
+      io.rewind
+      reader = Zstandard.inflate(io.read)
+      io.rewind
+      assert_equal(chunk.read, reader)
+      io.close
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.3.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-01 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstandard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -143,10 +157,12 @@ files:
 - lib/fluent/plugin/webhdfs_compressor_lzo_command.rb
 - lib/fluent/plugin/webhdfs_compressor_snappy.rb
 - lib/fluent/plugin/webhdfs_compressor_text.rb
+- lib/fluent/plugin/webhdfs_compressor_zstd.rb
 - test/helper.rb
 - test/plugin/test_compressor.rb
 - test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
+- test/plugin/test_zstd_compressor.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses:
 - Apache-2.0
@@ -175,3 +191,4 @@ test_files:
 - test/plugin/test_compressor.rb
 - test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
+- test/plugin/test_zstd_compressor.rb