fluent-plugin-webhdfs 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c6a284ca4dbae16f8154b790f6201d37f75b27a1
-  data.tar.gz: bbdd1477dd222f2a44308e1afcb7ec1ddd0f57ac
+  metadata.gz: a23088f567c4fe47907a5e411a3af7eb0d380e38
+  data.tar.gz: b33cc9662b3b56734bb0a82681e0925c41167be3
 SHA512:
-  metadata.gz: 986dd5af2013f754606db0a51abbfd1c4870e7e4c2cebf759bb27c02d9e73f1b7c1c1f2b5d57a53a5578ae65febcc20bb4022dcabbcef90c0ce40f34a660d433
-  data.tar.gz: 8768473997ac56b9af81c517e671a3c22ecf5048a841cc1b119c53ac6952b133751c7206b55237d06fc8268d152ccd454559360f270943172d6d105cc6ba9d5a
+  metadata.gz: 0611d377387664c3de86ba93ac030450e593ffaf52cac32ea819191efa2e8544e3348cf6715b2df8d776a439ca54d2cdba40fef784237f1a78fb8038c40d131a
+  data.tar.gz: ab4b691ae3ed374d5fbec3f9f6ab8605f9a7563f520ac40fc66c7c7d842fabc784f04e54a92eaccc9cb6a7e6eedc6ed474359e9695d11cfedf1e5da4514f92d8

data/.travis.yml

@@ -11,6 +11,11 @@ branches:
   only:
     - master
 
+addons:
+  apt:
+    packages:
+      - libsnappy-dev
+
 before_install:
   - gem update bundler
 

data/README.md

@@ -127,10 +127,10 @@ If you want to compress data before storing it:
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H
-      compress gzip  # or 'bzip2', 'lzo_command'
+      compress gzip  # or 'bzip2', 'snappy', 'lzo_command'
     </match>
 
-Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `.lzo`).
+Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`).
 
 ### Namenode HA / Auto retry for WebHDFS known errors
 

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.5.0"
+  gem.version       = "0.5.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
   gem.add_runtime_dependency "bzip2-ffi"
+  gem.add_runtime_dependency "snappy", '>= 0.0.13'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -85,7 +85,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   desc 'Use kerberos authentication or not'
   config_param :kerberos, :bool, :default => false
 
-  SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'lzo_command']
+  SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'snappy', 'lzo_command', 'text']
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :default => nil do |val|
     unless SUPPORTED_COMPRESS.include? val
@@ -357,4 +357,5 @@ end
 require 'fluent/plugin/webhdfs_compressor_text'
 require 'fluent/plugin/webhdfs_compressor_gzip'
 require 'fluent/plugin/webhdfs_compressor_bzip2'
+require 'fluent/plugin/webhdfs_compressor_snappy'
 require 'fluent/plugin/webhdfs_compressor_lzo_command'

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -0,0 +1,21 @@
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
+    class SnappyCompressor < Compressor
+      WebHDFSOutput.register_compressor('snappy', self)
+
+      def initialize(options = {})
+        require "snappy"
+      end
+
+      def ext
+        ".sz"
+      end
+
+      def compress(chunk, tmp)
+        w = Snappy::Writer.new(tmp)
+        chunk.write_to(w)
+        w.close
+      end
+    end
+  end
+end

data/test/plugin/test_compressor.rb

@@ -0,0 +1,29 @@
+require "helper"
+require "fluent/plugin/buf_memory"
+require "snappy"
+
+class CompressorTest < Test::Unit::TestCase
+  class Snappy < self
+    def setup
+      @compressor = Fluent::WebHDFSOutput::SnappyCompressor.new
+    end
+
+    def test_ext
+      assert_equal(".sz", @compressor.ext)
+    end
+
+    def test_compress
+      chunk = Fluent::MemoryBufferChunk.new("test")
+      chunk << "hello snappy\n" * 32 * 1024
+      io = Tempfile.new("snappy-")
+      @compressor.compress(chunk, io)
+      io.open
+      assert(chunk.size > io.read.bytesize)
+      io.rewind
+      reader = ::Snappy::Reader.new(io)
+      assert_equal(chunk.read, reader.read)
+      io.close
+    end
+  end
+end
+

data/test/plugin/test_out_webhdfs.rb

@@ -69,6 +69,7 @@ kerberos true
 
     data(gzip: ['gzip', Fluent::WebHDFSOutput::GzipCompressor],
          bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor],
+         snappy: ['snappy', Fluent::WebHDFSOutput::SnappyCompressor],
          lzo: ['lzo_command', Fluent::WebHDFSOutput::LZOCommandCompressor])
     def test_compress(data)
       compress_type, compressor_class = data

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-03 00:00:00.000000000 Z
+date: 2016-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.13
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -142,8 +156,10 @@ files:
 - lib/fluent/plugin/webhdfs_compressor_bzip2.rb
 - lib/fluent/plugin/webhdfs_compressor_gzip.rb
 - lib/fluent/plugin/webhdfs_compressor_lzo_command.rb
+- lib/fluent/plugin/webhdfs_compressor_snappy.rb
 - lib/fluent/plugin/webhdfs_compressor_text.rb
 - test/helper.rb
+- test/plugin/test_compressor.rb
 - test/plugin/test_out_webhdfs.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses:
@@ -171,5 +187,6 @@ specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
 test_files:
 - test/helper.rb
+- test/plugin/test_compressor.rb
 - test/plugin/test_out_webhdfs.rb
 has_rdoc: