fluent-plugin-webhdfs 1.3.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6318fcc2f54eb64be7d99978a04b1cad30e740734ed0dafdf1441cf65988f1ae
4
- data.tar.gz: 1d49b9c8b009a9a6078c4f9e9fe08b79832510cc780e50344ea72e8493c8e5c1
3
+ metadata.gz: 64bdfcd21e62ec4f83f7807b7b7a60e51f0ed68f2d015f39424dd47b76473be3
4
+ data.tar.gz: cbf248448c4eaa9b35689076ac0010753af7eb138d24cd377c43bcf5dfd46595
5
5
  SHA512:
6
- metadata.gz: 16f6c451fe30d09b33255dcac7ed8714beb0c157fee3bdce8e041b1de9c0aa032910b8cf21ac1effd8dddbea4240a44c5e68e3a83275e386fd881dbb22c75b1d
7
- data.tar.gz: c59533a82d5c46e9ea178402ede6852aa4d8fa7375981433b7f5b16311a55bb4e58234a0e28b657e318b690946c28622bfbda5da6b4318549342a394d6b8c15a
6
+ metadata.gz: d76806177dc1ff5e33dbf4fcc0e1fd0b3a47bd1a4fa1e51287a033ba4ccb94c750a540f735e1144be0bd502629c05ee19d74462ab2fe6831979d378b35686d20
7
+ data.tar.gz: e7d195870652a72c6d3edea039336742e3564a01ac9871036ddc4dcb3d0eb58bb4b9c73ac6598ea2712ec1ec17870e9ee6d091a2a89204688fb3935aa5ccc34b
data/README.md CHANGED
@@ -169,6 +169,20 @@ Note that you have to install additional gem for several compress algorithms:
169
169
 
170
170
  Note that zstd will require installation of the libzstd native library. See the [zstandard-ruby](https://github.com/msievers/zstandard-ruby#examples-for-installing-libzstd) repo for infomration on the required packages for your operating system.
171
171
 
172
+ If you want to explicitly specify file extensions in HDFS (override default compressor extensions):
173
+
174
+ <match access.**>
175
+ @type webhdfs
176
+ host namenode.your.cluster.local
177
+ port 50070
178
+ path /path/on/hdfs/access.log.%Y%m%d_%H
179
+ compress snappy
180
+ extension ".snappy"
181
+ </match>
182
+
183
+ With this configuration paths in HDFS will be like `/path/on/hdfs/access.log.20201003_12.snappy`.
184
+ This one may be useful when (for example) you need to use snappy codec but `.sz` files are not recognized as snappy files in HDFS.
185
+
172
186
  ### Namenode HA / Auto retry for WebHDFS known errors
173
187
 
174
188
  `fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = "fluent-plugin-webhdfs"
5
- gem.version = "1.3.0"
5
+ gem.version = "1.3.1"
6
6
  gem.authors = ["TAGOMORI Satoshi"]
7
7
  gem.email = ["tagomoris@gmail.com"]
8
8
  gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -71,6 +71,9 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
71
71
  desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
72
72
  config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
73
73
 
74
+ desc 'HDFS file extensions (overrides default compressor extensions)'
75
+ config_param :extension, :string, default: nil
76
+
74
77
  config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
75
78
  config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
76
79
  config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
@@ -319,7 +322,8 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
319
322
  else
320
323
  extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk)
321
324
  end
322
- hdfs_path = "#{hdfs_path}#{@compressor.ext}"
325
+ hdfs_ext = @extension || @compressor.ext
326
+ hdfs_path = "#{hdfs_path}#{hdfs_ext}"
323
327
  if @replace_random_uuid
324
328
  uuid_random = SecureRandom.uuid
325
329
  hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)
@@ -126,6 +126,26 @@ class WebHDFSOutputTest < Test::Unit::TestCase
126
126
  assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
127
127
  assert_equal compress_type, d.instance.compress
128
128
  assert_equal compressor_class, d.instance.compressor.class
129
+
130
+ time = event_time("2020-10-03 15:07:00 +0300")
131
+ metadata = d.instance.metadata("test", time, {})
132
+ chunk = d.instance.buffer.generate_chunk(metadata)
133
+ assert_equal "/hdfs/path/file.20201003.1507.log#{d.instance.compressor.ext}", d.instance.generate_path(chunk)
134
+ end
135
+
136
+ def test_explicit_extensions
137
+ conf = config_element(
138
+ "ROOT", "", {
139
+ "host" => "namenode.local",
140
+ "path" => "/hdfs/path/file.%Y%m%d.log",
141
+ "compress" => "snappy",
142
+ "extension" => ".snappy"
143
+ })
144
+ d = create_driver(conf)
145
+ time = event_time("2020-10-07 15:15:00 +0300")
146
+ metadata = d.instance.metadata("test", time, {})
147
+ chunk = d.instance.buffer.generate_chunk(metadata)
148
+ assert_equal "/hdfs/path/file.20201007.log.snappy", d.instance.generate_path(chunk)
129
149
  end
130
150
 
131
151
  def test_placeholders_old_style
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-webhdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-10-20 00:00:00.000000000 Z
11
+ date: 2020-11-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake