fluent-plugin-webhdfs 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6318fcc2f54eb64be7d99978a04b1cad30e740734ed0dafdf1441cf65988f1ae
-  data.tar.gz: 1d49b9c8b009a9a6078c4f9e9fe08b79832510cc780e50344ea72e8493c8e5c1
+  metadata.gz: 64bdfcd21e62ec4f83f7807b7b7a60e51f0ed68f2d015f39424dd47b76473be3
+  data.tar.gz: cbf248448c4eaa9b35689076ac0010753af7eb138d24cd377c43bcf5dfd46595
 SHA512:
-  metadata.gz: 16f6c451fe30d09b33255dcac7ed8714beb0c157fee3bdce8e041b1de9c0aa032910b8cf21ac1effd8dddbea4240a44c5e68e3a83275e386fd881dbb22c75b1d
-  data.tar.gz: c59533a82d5c46e9ea178402ede6852aa4d8fa7375981433b7f5b16311a55bb4e58234a0e28b657e318b690946c28622bfbda5da6b4318549342a394d6b8c15a
+  metadata.gz: d76806177dc1ff5e33dbf4fcc0e1fd0b3a47bd1a4fa1e51287a033ba4ccb94c750a540f735e1144be0bd502629c05ee19d74462ab2fe6831979d378b35686d20
+  data.tar.gz: e7d195870652a72c6d3edea039336742e3564a01ac9871036ddc4dcb3d0eb58bb4b9c73ac6598ea2712ec1ec17870e9ee6d091a2a89204688fb3935aa5ccc34b

data/README.md

@@ -169,6 +169,20 @@ Note that you have to install additional gem for several compress algorithms:
 
 Note that zstd will require installation of the libzstd native library. See the [zstandard-ruby](https://github.com/msievers/zstandard-ruby#examples-for-installing-libzstd) repo for infomration on the required packages for your operating system.
 
+If you want to explicitly specify file extensions in HDFS (override default compressor extensions):
+
+    <match access.**>
+      @type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H
+      compress snappy
+      extension ".snappy"
+    </match>
+
+With this configuration paths in HDFS will be like `/path/on/hdfs/access.log.20201003_12.snappy`.
+This one may be useful when (for example) you need to use snappy codec but `.sz` files are not recognized as snappy files in HDFS.
+
 ### Namenode HA / Auto retry for WebHDFS known errors
 
 `fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.3.0"
+  gem.version       = "1.3.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}

data/lib/fluent/plugin/out_webhdfs.rb

@@ -71,6 +71,9 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
 
+  desc 'HDFS file extensions (overrides default compressor extensions)'
+  config_param :extension, :string, default: nil
+
   config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
   config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
   config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
@@ -319,7 +322,8 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
                 else
                   extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk)
                 end
-    hdfs_path = "#{hdfs_path}#{@compressor.ext}"
+    hdfs_ext = @extension || @compressor.ext
+    hdfs_path = "#{hdfs_path}#{hdfs_ext}"
     if @replace_random_uuid
       uuid_random = SecureRandom.uuid
       hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)

data/test/plugin/test_out_webhdfs.rb

@@ -126,6 +126,26 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
       assert_equal compress_type, d.instance.compress
       assert_equal compressor_class, d.instance.compressor.class
+
+      time = event_time("2020-10-03 15:07:00 +0300")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.20201003.1507.log#{d.instance.compressor.ext}", d.instance.generate_path(chunk)
+    end
+
+    def test_explicit_extensions
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d.log",
+          "compress" => "snappy",
+          "extension" => ".snappy"
+        })
+      d = create_driver(conf)
+      time = event_time("2020-10-07 15:15:00 +0300")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal "/hdfs/path/file.20201007.log.snappy", d.instance.generate_path(chunk)
     end
 
     def test_placeholders_old_style

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake