fluent-plugin-webhdfs 1.2.2 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4637e563a9e04c877cd6d897d013ac57a63d1a80
-  data.tar.gz: 305a8e89b883175c47b4fc9811fd1a27f9773141
+SHA256:
+  metadata.gz: 64bdfcd21e62ec4f83f7807b7b7a60e51f0ed68f2d015f39424dd47b76473be3
+  data.tar.gz: cbf248448c4eaa9b35689076ac0010753af7eb138d24cd377c43bcf5dfd46595
 SHA512:
-  metadata.gz: ded6f6b4ba2ca28fa49be0a4a23127ec643d1adfc046c7cb14d37cc91ad90705252ac7cc986439c0793849c876ffcf88c5b6abc21a4ba9f7284c6933e07a8632
-  data.tar.gz: 1e1d5b936463bfab84baeec2cf9b6064dcd25890712f8d315b5a18457dc87087c4270a2a2f1cdf0a9e390c91345bb9ac1eb739088644537c0a6bae1564cf1325
+  metadata.gz: d76806177dc1ff5e33dbf4fcc0e1fd0b3a47bd1a4fa1e51287a033ba4ccb94c750a540f735e1144be0bd502629c05ee19d74462ab2fe6831979d378b35686d20
+  data.tar.gz: e7d195870652a72c6d3edea039336742e3564a01ac9871036ddc4dcb3d0eb58bb4b9c73ac6598ea2712ec1ec17870e9ee6d091a2a89204688fb3935aa5ccc34b

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+vendor/

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-webhdfs
 
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-webhdfs.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-webhdfs)
+
 [Fluentd](http://fluentd.org/) output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
 
 "webhdfs" output plugin formats data into plain text, and store it as files on HDFS. This plugin supports:
@@ -155,14 +157,31 @@ If you want to compress data before storing it:
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H
-      compress gzip  # or 'bzip2', 'snappy', 'lzo_command'
+      compress gzip  # or 'bzip2', 'snappy', 'lzo_command', 'zstd'
     </match>
 
-Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`).
+Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`, `.zst`).
 Note that you have to install additional gem for several compress algorithms:
 
 - snappy: install snappy gem
 - bzip2: install bzip2-ffi gem
+- zstd: install zstandard gem
+
+Note that zstd will require installation of the libzstd native library. See the [zstandard-ruby](https://github.com/msievers/zstandard-ruby#examples-for-installing-libzstd) repo for infomration on the required packages for your operating system.
+
+If you want to explicitly specify file extensions in HDFS (override default compressor extensions):
+
+    <match access.**>
+      @type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H
+      compress snappy
+      extension ".snappy"
+    </match>
+
+With this configuration paths in HDFS will be like `/path/on/hdfs/access.log.20201003_12.snappy`.
+This one may be useful when (for example) you need to use snappy codec but `.sz` files are not recognized as snappy files in HDFS.
 
 ### Namenode HA / Auto retry for WebHDFS known errors
 

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.2.2"
+  gem.version       = "1.3.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -21,6 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "appraisal"
   gem.add_development_dependency "snappy", '>= 0.0.13'
   gem.add_development_dependency "bzip2-ffi"
-  gem.add_runtime_dependency "fluentd", '>= 0.14.4'
+  gem.add_development_dependency "zstandard"
+  gem.add_runtime_dependency "fluentd", '>= 0.14.22'
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -67,10 +67,13 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   desc 'kerberos keytab file'
   config_param :kerberos_keytab, :string, default: nil
 
-  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :text]
+  SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :zstd, :text]
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
 
+  desc 'HDFS file extensions (overrides default compressor extensions)'
+  config_param :extension, :string, default: nil
+
   config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
   config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
   config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
@@ -98,20 +101,27 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   end
 
   def configure(conf)
-    compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
-
+    # #compat_parameters_convert ignore time format in conf["path"],
+    # so check conf["path"] and overwrite the default value later if needed
     timekey = case conf["path"]
               when /%S/ then 1
               when /%M/ then 60
               when /%H/ then 3600
               else 86400
               end
+    if buffer_config = conf.elements(name: "buffer").first
+      timekey = buffer_config["timekey"] || timekey 
+    end
+
+    compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
+
     if conf.elements(name: "buffer").empty?
       e = Fluent::Config::Element.new("buffer", "time", {}, [])
       conf.elements << e
     end
     buffer_config = conf.elements(name: "buffer").first
-    buffer_config["timekey"] = timekey unless buffer_config["timekey"]
+    # explicitly set timekey
+    buffer_config["timekey"] = timekey
 
     compat_parameters_convert_plaintextformatter(conf)
     verify_config_placeholders_in_path!(conf)
@@ -308,11 +318,12 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
 
   def generate_path(chunk)
     hdfs_path = if @append
-                  extract_placeholders(@path, chunk.metadata)
+                  extract_placeholders(@path, chunk)
                 else
-                  extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk.metadata)
+                  extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk)
                 end
-    hdfs_path = "#{hdfs_path}#{@compressor.ext}"
+    hdfs_ext = @extension || @compressor.ext
+    hdfs_path = "#{hdfs_path}#{hdfs_ext}"
     if @replace_random_uuid
       uuid_random = SecureRandom.uuid
       hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)
@@ -512,3 +523,4 @@ require 'fluent/plugin/webhdfs_compressor_gzip'
 require 'fluent/plugin/webhdfs_compressor_bzip2'
 require 'fluent/plugin/webhdfs_compressor_snappy'
 require 'fluent/plugin/webhdfs_compressor_lzo_command'
+require 'fluent/plugin/webhdfs_compressor_zstd'

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -16,9 +16,10 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
-        w = Snappy::Writer.new(tmp)
-        chunk.write_to(w)
-        w.close
+        Snappy::Writer.new(tmp) do |w|
+          w << chunk.read
+          w.flush
+        end
       end
     end
   end

data/lib/fluent/plugin/webhdfs_compressor_zstd.rb

@@ -0,0 +1,24 @@
+module Fluent::Plugin
+  class WebHDFSOutput < Output
+    class ZstdCompressor < Compressor
+      WebHDFSOutput.register_compressor('zstd', self)
+
+      def initialize(options = {})
+        begin
+          require "zstandard"
+        rescue LoadError
+          raise Fluent::ConfigError, "Install zstandard gem before use of zstd compressor"
+        end
+      end
+
+      def ext
+        ".zst"
+      end
+
+      def compress(chunk, tmp)
+        tmp.binmode
+        tmp.write Zstandard.deflate(chunk.read)
+      end
+    end
+  end
+end

data/test/plugin/test_out_webhdfs.rb

@@ -126,6 +126,26 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
       assert_equal compress_type, d.instance.compress
       assert_equal compressor_class, d.instance.compressor.class
+
+      time = event_time("2020-10-03 15:07:00 +0300")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.20201003.1507.log#{d.instance.compressor.ext}", d.instance.generate_path(chunk)
+    end
+
+    def test_explicit_extensions
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d.log",
+          "compress" => "snappy",
+          "extension" => ".snappy"
+        })
+      d = create_driver(conf)
+      time = event_time("2020-10-07 15:15:00 +0300")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.20201007.log.snappy", d.instance.generate_path(chunk)
     end
 
     def test_placeholders_old_style
@@ -207,6 +227,21 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal 1, d.instance.buffer_config.timekey
       assert_equal "/hdfs/path/file.20120718.log", d.instance.generate_path(chunk)
     end
+
+    def test_time_key_without_buffer_section
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d-%M.log"
+        }
+      )
+      d = create_driver(conf)
+      time = event_time("2012-07-18 15:03:00 +0900")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal 60, d.instance.buffer_config.timekey
+      assert_equal "/hdfs/path/file.20120718-03.log", d.instance.generate_path(chunk)
+    end
   end
 
   sub_test_case "using format subsection" do

data/test/plugin/test_zstd_compressor.rb

@@ -0,0 +1,55 @@
+require "helper"
+require "fluent/plugin/buf_memory"
+require 'zstandard'
+
+class ZstdCompressorTest < Test::Unit::TestCase
+  class Zstd < self
+
+    CONFIG = %[
+      host namenode.local
+      path /hdfs/path/file.%Y%m%d.log
+    ]
+
+    def setup
+      omit unless Object.const_defined?(:Zstandard)
+      Fluent::Test.setup
+      @compressor = Fluent::Plugin::WebHDFSOutput::ZstdCompressor.new
+    end
+
+    def create_driver(conf = CONFIG)
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::WebHDFSOutput).configure(conf)
+    end
+
+    def test_ext
+      assert_equal(".zst", @compressor.ext)
+    end
+
+    def test_compress
+      d = create_driver
+      if d.instance.respond_to?(:buffer)
+        buffer = d.instance.buffer
+      else
+        buffer = d.instance.instance_variable_get(:@buffer)
+      end
+
+      if buffer.respond_to?(:generate_chunk)
+        chunk = buffer.generate_chunk("test")
+        chunk.concat("hello gzip\n" * 32 * 1024, 1)
+      else
+        chunk = buffer.new_chunk("test")
+        chunk << "hello gzip\n" * 32 * 1024
+      end
+
+      io = Tempfile.new("zstd-")
+      @compressor.compress(chunk, io)
+      assert !io.closed?
+      chunk_bytesize = chunk.respond_to?(:bytesize) ? chunk.bytesize : chunk.size
+      assert(chunk_bytesize > io.read.bytesize)
+      io.rewind
+      reader = Zstandard.inflate(io.read)
+      io.rewind
+      assert_equal(chunk.read, reader)
+      io.close
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-23 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -94,20 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstandard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.4
+        version: 0.14.22
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.4
+        version: 0.14.22
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -143,10 +157,12 @@ files:
 - lib/fluent/plugin/webhdfs_compressor_lzo_command.rb
 - lib/fluent/plugin/webhdfs_compressor_snappy.rb
 - lib/fluent/plugin/webhdfs_compressor_text.rb
+- lib/fluent/plugin/webhdfs_compressor_zstd.rb
 - test/helper.rb
 - test/plugin/test_compressor.rb
 - test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
+- test/plugin/test_zstd_compressor.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses:
 - Apache-2.0
@@ -166,8 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
@@ -176,3 +191,4 @@ test_files:
 - test/plugin/test_compressor.rb
 - test/plugin/test_gzip_compressor.rb
 - test/plugin/test_out_webhdfs.rb
+- test/plugin/test_zstd_compressor.rb