fluent-plugin-webhdfs 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a23088f567c4fe47907a5e411a3af7eb0d380e38
-  data.tar.gz: b33cc9662b3b56734bb0a82681e0925c41167be3
+  metadata.gz: 6037c3f039431852ee3f19e0fde88a37a00636c3
+  data.tar.gz: 4dbc0b1bbaf6d921bcd4331392ed0e5ac980e28b
 SHA512:
-  metadata.gz: 0611d377387664c3de86ba93ac030450e593ffaf52cac32ea819191efa2e8544e3348cf6715b2df8d776a439ca54d2cdba40fef784237f1a78fb8038c40d131a
-  data.tar.gz: ab4b691ae3ed374d5fbec3f9f6ab8605f9a7563f520ac40fc66c7c7d842fabc784f04e54a92eaccc9cb6a7e6eedc6ed474359e9695d11cfedf1e5da4514f92d8
+  metadata.gz: c4f58538cb9ee701da053b7a3d5a3c60f338e25a6de00980b6ed29817c08d37910d84bf9bb314e1936353aa45cd282d10601e8e8fea897c65583513b305ac58b
+  data.tar.gz: 7cc16b12bd77d47b50b0368c7f7dbc58188f9a8e980acb41e1d45428a48f47aade6d3b357c113cdeee8410e3784747300df17707c28f4eaf9107cfd0376c7e24

data/.travis.yml

@@ -24,3 +24,11 @@ script: bundle exec rake test
 gemfile:
   - Gemfile
   - gemfiles/fluentd_v0.12.gemfile
+  - gemfiles/fluentd_v0.14.gemfile
+
+matrix:
+  exclude:
+    - rvm: 2.0.0
+      gemfile: Gemfile
+    - rvm: 2.0.0
+      gemfile: gemfiles/fluentd_v0.14.gemfile

data/Appraisals

@@ -1,4 +1,10 @@
 appraise "fluentd v0.12" do
   gem "fluentd", "~>0.12.0"
+  gem "snappy"
+end
+
+appraise "fluentd v0.14" do
+  gem "fluentd", "~>0.14.0"
+  gem "snappy"
 end
 

data/README.md

@@ -131,6 +131,7 @@ If you want to compress data before storing it:
     </match>
 
 Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`).
+Note that you have to install snappy gem if you want to  set `compress snappy`.
 
 ### Namenode HA / Auto retry for WebHDFS known errors
 

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.5.1"
+  gem.version       = "0.5.2"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -18,10 +18,10 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
   gem.add_development_dependency "appraisal"
+  gem.add_development_dependency "snappy", '>= 0.0.13'
   gem.add_runtime_dependency "fluentd", '>= 0.10.59'
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
   gem.add_runtime_dependency "bzip2-ffi"
-  gem.add_runtime_dependency "snappy", '>= 0.0.13'
 end

data/gemfiles/fluentd_v0.14.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~>0.14.0"
+
+gemspec :path => "../"

data/lib/fluent/plugin/out_webhdfs.rb

@@ -127,6 +127,8 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
     begin
       @compressor = COMPRESSOR_REGISTRY.lookup(@compress || 'text').new
+    rescue Fluent::ConfigError
+      raise
     rescue
       $log.warn "#{@comress} not found. Use 'text' instead"
       @compressor = COMPRESSOR_REGISTRY.lookup('text').new

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -4,7 +4,11 @@ module Fluent
       WebHDFSOutput.register_compressor('snappy', self)
 
       def initialize(options = {})
-        require "snappy"
+        begin
+          require "snappy"
+        rescue LoadError
+          raise Fluent::ConfigError, "Install snappy before use snappy compressor"
+        end
       end
 
       def ext

data/test/plugin/test_compressor.rb

@@ -1,24 +1,53 @@
 require "helper"
 require "fluent/plugin/buf_memory"
-require "snappy"
+begin
+  require "snappy"
+rescue LoadError
+end
 
 class CompressorTest < Test::Unit::TestCase
   class Snappy < self
+
+    CONFIG = %[
+      host namenode.local
+      path /hdfs/path/file.%Y%m%d.log
+    ]
+
     def setup
+      omit unless Object.const_defined?(:Snappy)
+      Fluent::Test.setup
       @compressor = Fluent::WebHDFSOutput::SnappyCompressor.new
     end
 
+    def create_driver(conf=CONFIG,tag='test')
+      Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
+    end
+
     def test_ext
       assert_equal(".sz", @compressor.ext)
     end
 
     def test_compress
-      chunk = Fluent::MemoryBufferChunk.new("test")
-      chunk << "hello snappy\n" * 32 * 1024
+      d = create_driver
+      if d.instance.respond_to?(:buffer)
+        buffer = d.instance.buffer
+      else
+        buffer = d.instance.instance_variable_get(:@buffer)
+      end
+
+      if buffer.respond_to?(:generate_chunk)
+        chunk = buffer.generate_chunk("test")
+        chunk.concat("hello snappy\n" * 32 * 1024, 1)
+      else
+        chunk = buffer.new_chunk("test")
+        chunk << "hello snappy\n" * 32 * 1024
+      end
+
       io = Tempfile.new("snappy-")
       @compressor.compress(chunk, io)
       io.open
-      assert(chunk.size > io.read.bytesize)
+      chunk_bytesize = chunk.respond_to?(:bytesize) ? chunk.bytesize : chunk.size
+      assert(chunk_bytesize > io.read.bytesize)
       io.rewind
       reader = ::Snappy::Reader.new(io)
       assert_equal(chunk.read, reader.read)

data/test/plugin/test_out_webhdfs.rb

@@ -73,11 +73,15 @@ kerberos true
          lzo: ['lzo_command', Fluent::WebHDFSOutput::LZOCommandCompressor])
     def test_compress(data)
       compress_type, compressor_class = data
-      d = create_driver %[
+      begin
+        d = create_driver %[
 namenode server.local:14000
 path /hdfs/path/file.%Y%m%d.%H%M.log
 compress #{compress_type}
-]
+        ]
+      rescue Fluent::ConfigError => ex
+        omit ex.message
+      end
       assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
       assert_equal 14000, d.instance.instance_eval{ @namenode_port }
       assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-05 00:00:00.000000000 Z
+date: 2016-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.13
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.13
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +136,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: snappy
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.13
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.13
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -152,6 +152,7 @@ files:
 - Rakefile
 - fluent-plugin-webhdfs.gemspec
 - gemfiles/fluentd_v0.12.gemfile
+- gemfiles/fluentd_v0.14.gemfile
 - lib/fluent/plugin/out_webhdfs.rb
 - lib/fluent/plugin/webhdfs_compressor_bzip2.rb
 - lib/fluent/plugin/webhdfs_compressor_gzip.rb