fluent-plugin-webhdfs 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e67f69a6f94c64ebe9b463b262111626a5cfb0e3
-  data.tar.gz: 9786c0456e296851caf6001898c8e30751724b04
+  metadata.gz: 5e43e061acd7552d5a0e482bf7ced1e319d1d107
+  data.tar.gz: 934ac37c1a6d985261ae573ce0724546481269ff
 SHA512:
-  metadata.gz: 6766521d3edaf773135c7d948ba0f8989241535795f7a38d61961c3dd9823df83955c0e4f817b029600c8e83495e24443015d2b863f68dd911dc6ea7ca5f8274
-  data.tar.gz: 3834bcb691d9d5c86df91169f44f77a9aaad09754d849919138be5ce403271b3a168c3620a0bbafc80bd8f3a2553cc0b3fd97bc14e794c0637dd1cc06dc3c7ec
+  metadata.gz: be9c06348692d4f0506f0e7d8caa1fc4d6bfa4d683c1fdb0daa3fd87d4657ddbebaff4380c3fc03405030836cabc2fb856e02754aa8bccff1ddd5f7c7b49980f
+  data.tar.gz: fee1d87a57a64ea65ca14a2f3f7765eb115bd362d504f0ccade232132a384b4d8f599ccd264c8cdcb2f326c4a8062b7c5e456e4cf11c8cc7fab5d78dadf09e3a

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - 1.9.2
   - 1.9.3
   - 2.0.0
+  - 2.1.1
 
 branches:
   only:

data/README.md

@@ -120,6 +120,18 @@ With kerberos authentication:
       kerberos true
     </match>
 
+If you want to compress data before storing it:
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H
+      compress gzip  # currently only support gzip
+    </match>
+
+Note that if you set `compress gzip`, then the suffix `.gz` will be added to path.
+
 ### Namenode HA / Auto retry for WebHDFS known errors
 
 `fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.4.0"
+  gem.version       = "0.4.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}

data/lib/fluent/plugin/out_webhdfs.rb

@@ -50,12 +50,20 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     when 'peer'
       :peer
     else
-      raise ConfigError, "unexpected parameter on ssl_verify_mode: #{val}"
+      raise Fluent::ConfigError, "unexpected parameter on ssl_verify_mode: #{val}"
     end
   end
 
   config_param :kerberos, :bool, :default => false
 
+  SUPPORTED_COMPRESS = ['gzip']
+  config_param :compress, :default => nil do |val|
+    unless SUPPORTED_COMPRESS.include? val
+      raise Fluent::ConfigError, "unsupported compress: #{val}"
+    end
+    val
+  end
+
   CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
 
   def initialize
@@ -216,16 +224,52 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
-  def write(chunk)
+  def generate_path(chunk)
     hdfs_path = if @append
                   path_format(chunk.key)
                 else
                   path_format(chunk.key).gsub(CHUNK_ID_PLACE_HOLDER, chunk_unique_id_to_str(chunk.unique_id))
                 end
+    if @compress
+      case @compress
+      when 'gzip'
+        hdfs_path = "#{hdfs_path}.gz"
+      end
+    end
+    hdfs_path
+  end
+
+  def compress_context(chunk, &block)
+    case @compress
+    when 'gzip'
+      require 'zlib'
+      require 'tempfile'
+      tmp = Tempfile.new("webhdfs-")
+      begin
+        w = Zlib::GzipWriter.new(tmp)
+        chunk.write_to(w)
+        w.close
+        tmp.close
+        tmp.open
+        yield tmp
+      ensure
+        tmp.close(true) rescue nil
+      end
+    end
+  end
+
+  def write(chunk)
+    hdfs_path = generate_path(chunk)
 
     failovered = false
     begin
-      send_data(hdfs_path, chunk.read)
+      if @compress
+        compress_context(chunk) do |data|
+          send_data(hdfs_path, data)
+        end
+      else
+        send_data(hdfs_path, chunk.read)
+      end
     rescue => e
       log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
 

data/test/plugin/test_out_webhdfs.rb

@@ -57,6 +57,17 @@ kerberos true
     assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
     assert_equal :peer, d.instance.ssl_verify_mode
     assert_equal true, d.instance.kerberos
+
+    d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+compress gzip
+]
+    assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
+    assert_equal 14000, d.instance.instance_eval{ @namenode_port }
+    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+    assert_equal 'gzip', d.instance.compress
   end
 
   def test_configure_placeholders
@@ -90,4 +101,23 @@ path /hdfs/path/file.%Y%m%d.%H%M.log
           ]
     end
   end
+
+  def test_invalid_configure
+    assert_raise Fluent::ConfigError do
+      create_driver %[
+        namenode server.local:14000
+        path /hdfs/path/file.%Y%m%d.%H%M.log
+        ssl true
+        ssl_verify_mode invalid
+      ]
+    end
+    assert_raise Fluent::ConfigError do
+      create_driver %[
+        namenode server.local:14000
+        path /hdfs/path/file.%Y%m%d.%H%M.log
+        compress invalid
+      ]
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-30 00:00:00.000000000 Z
+date: 2014-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake