fluent-plugin-webhdfs 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61f1146fdfa68de12937082f2321f097e4aa5858
-  data.tar.gz: 502673646e6db2db20973079a4c2056515b7de11
+  metadata.gz: c6a284ca4dbae16f8154b790f6201d37f75b27a1
+  data.tar.gz: bbdd1477dd222f2a44308e1afcb7ec1ddd0f57ac
 SHA512:
-  metadata.gz: b634dd49ac2d1e3d552e7dbda396cc5aca47c0cd090219c5464a3a834ec9ffd5c0030d013474e4b4ac83870252f45af567788d97e0671113278d41bd168b969a
-  data.tar.gz: 90dfe3fc5e45a3667df905874607875cba4cd18e49e22cc05318dd4141ec1e1cec3d36d933ac15634d780ef46ab75565d63fea6ffcc5c3678b901ced071042c4
+  metadata.gz: 986dd5af2013f754606db0a51abbfd1c4870e7e4c2cebf759bb27c02d9e73f1b7c1c1f2b5d57a53a5578ae65febcc20bb4022dcabbcef90c0ce40f34a660d433
+  data.tar.gz: 8768473997ac56b9af81c517e671a3c22ecf5048a841cc1b119c53ac6952b133751c7206b55237d06fc8268d152ccd454559360f270943172d6d105cc6ba9d5a

data/.travis.yml

@@ -1,9 +1,11 @@
+sudo: false
 language: ruby
 
 rvm:
   - 2.0.0
   - 2.1
   - 2.2
+  - 2.3.0
 
 branches:
   only:
@@ -13,3 +15,7 @@ before_install:
   - gem update bundler
 
 script: bundle exec rake test
+
+gemfile:
+  - Gemfile
+  - gemfiles/fluentd_v0.12.gemfile

data/Appraisals

@@ -0,0 +1,4 @@
+appraise "fluentd v0.12" do
+  gem "fluentd", "~>0.12.0"
+end
+

data/README.md

@@ -17,10 +17,10 @@ And you can specify output file path as 'path /path/to/dir/access.%Y%m%d.log', t
 
 ### WebHDFSOutput
 
-To store data by time,tag,json (same with 'type file') over WebHDFS:
+To store data by time,tag,json (same with '@type file') over WebHDFS:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -29,7 +29,7 @@ To store data by time,tag,json (same with 'type file') over WebHDFS:
 If you want JSON object only (without time or tag or both on header of lines), specify it by `output_include_time` or `output_include_tag` (default true):
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -40,7 +40,7 @@ If you want JSON object only (without time or tag or both on header of lines), s
 To specify namenode, `namenode` is also available:
 
     <match access.**>
-      type     webhdfs
+      @type     webhdfs
       namenode master.your.cluster.local:50070
       path     /path/on/hdfs/access.log.%Y%m%d_%H.log
     </match>
@@ -48,7 +48,7 @@ To specify namenode, `namenode` is also available:
 To store data as LTSV without time and tag over WebHDFS:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -58,7 +58,7 @@ To store data as LTSV without time and tag over WebHDFS:
 With username of pseudo authentication:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -68,7 +68,7 @@ With username of pseudo authentication:
 Store data over HttpFs (instead of WebHDFS):
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host httpfs.node.your.cluster.local
       port 14000
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -78,7 +78,7 @@ Store data over HttpFs (instead of WebHDFS):
 Store data as TSV (TAB separated values) of specified keys, without time, with tag (removed prefix 'access'):
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -96,7 +96,7 @@ If message doesn't have specified attribute, fluent-plugin-webhdfs outputs 'NULL
 With ssl:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -113,7 +113,7 @@ and [openssl](http://www.ruby-doc.org/stdlib-2.1.3/libdoc/openssl/rdoc/OpenSSL.h
 With kerberos authentication:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -123,21 +123,21 @@ With kerberos authentication:
 If you want to compress data before storing it:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H
-      compress gzip  # currently only support gzip
+      compress gzip  # or 'bzip2', 'lzo_command'
     </match>
 
-Note that if you set `compress gzip`, then the suffix `.gz` will be added to path.
+Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `.lzo`).
 
 ### Namenode HA / Auto retry for WebHDFS known errors
 
 `fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:
 
     <match access.**>
-      type             webhdfs
+      @type            webhdfs
       namenode         master1.your.cluster.local:50070
 	  standby_namenode master2.your.cluster.local:50070
       path             /path/on/hdfs/access.log.%Y%m%d_%H.log
@@ -146,7 +146,7 @@ Note that if you set `compress gzip`, then the suffix `.gz` will be added to pat
 And you can also specify to retry known hdfs errors (such like `LeaseExpiredException`) automatically. With this configuration, fluentd doesn't write logs for this errors if retry successed.
 
     <match access.**>
-      type               webhdfs
+      @type              webhdfs
       namenode           master1.your.cluster.local:50070
       path               /path/on/hdfs/access.log.%Y%m%d_%H.log
 	  retry_known_errors yes
@@ -162,7 +162,7 @@ You can use '${hostname}' or '${uuid:random}' placeholders in configuration for
 For hostname:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /log/access/%Y%m%d/${hostname}.log
@@ -171,7 +171,7 @@ For hostname:
 Or with random filename (to avoid duplicated file name only):
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /log/access/%Y%m%d/${uuid:random}.log
@@ -182,7 +182,7 @@ With configurations above, you can handle all of files of '/log/access/20120820/
 For high load cluster nodes, you can specify timeouts for HTTP requests.
 
     <match access.**>
-	  type webhdfs
+	  @type webhdfs
 	  namenode master.your.cluster.local:50070
       path /log/access/%Y%m%d/${hostname}.log
 	  open_timeout 180 # [sec] default: 30
@@ -196,7 +196,7 @@ With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status
 If you were usging unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       path /log/access/%Y%m%d/${hostname}.log
@@ -208,7 +208,7 @@ If you were usging unstable NameNodes and have wanted to ignore NameNode errors
 With unstable datanodes that frequently downs, appending over WebHDFS may produce broken files. In such cases, specify `append no` and `${chunk_id}` parameter.
 
     <match access.**>
-      type webhdfs
+      @type webhdfs
       host namenode.your.cluster.local
       port 50070
       

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.4.2"
+  gem.version       = "0.5.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -17,7 +17,8 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
-  gem.add_runtime_dependency "fluentd", '>= 0.10.53'
+  gem.add_development_dependency "appraisal"
+  gem.add_runtime_dependency "fluentd", '>= 0.10.59'
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'

data/gemfiles/fluentd_v0.12.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~>0.12.0"
+
+gemspec :path => "../"

data/lib/fluent/plugin/out_webhdfs.rb

@@ -85,7 +85,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   desc 'Use kerberos authentication or not'
   config_param :kerberos, :bool, :default => false
 
-  SUPPORTED_COMPRESS = ['gzip', 'bzip2']
+  SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'lzo_command']
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :default => nil do |val|
     unless SUPPORTED_COMPRESS.include? val
@@ -357,3 +357,4 @@ end
 require 'fluent/plugin/webhdfs_compressor_text'
 require 'fluent/plugin/webhdfs_compressor_gzip'
 require 'fluent/plugin/webhdfs_compressor_bzip2'
+require 'fluent/plugin/webhdfs_compressor_lzo_command'

data/lib/fluent/plugin/webhdfs_compressor_lzo_command.rb

@@ -0,0 +1,31 @@
+module Fluent
+  class WebHDFSOutput
+    class LZOCommandCompressor < Compressor
+      WebHDFSOutput.register_compressor('lzo_command', self)
+
+      config_param :command_parameter, :string, :default => '-qf1'
+
+      def configure(conf)
+        super
+        check_command('lzop', 'LZO')
+      end
+
+      def ext
+        '.lzo'
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-lzo-tmp-")
+        w.binmode
+        chunk.write_to(w)
+        w.close
+
+        # We don't check the return code because we can't recover lzop failure.
+        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/test/plugin/test_out_webhdfs.rb

@@ -68,7 +68,8 @@ kerberos true
     end
 
     data(gzip: ['gzip', Fluent::WebHDFSOutput::GzipCompressor],
-         bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor])
+         bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor],
+         lzo: ['lzo_command', Fluent::WebHDFSOutput::LZOCommandCompressor])
     def test_compress(data)
       compress_type, compressor_class = data
       d = create_driver %[

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -38,20 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.53
+        version: 0.10.59
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.53
+        version: 0.10.59
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-plaintextformatter
   requirement: !ruby/object:Gem::Requirement
@@ -117,14 +131,17 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- Appraisals
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-webhdfs.gemspec
+- gemfiles/fluentd_v0.12.gemfile
 - lib/fluent/plugin/out_webhdfs.rb
 - lib/fluent/plugin/webhdfs_compressor_bzip2.rb
 - lib/fluent/plugin/webhdfs_compressor_gzip.rb
+- lib/fluent/plugin/webhdfs_compressor_lzo_command.rb
 - lib/fluent/plugin/webhdfs_compressor_text.rb
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
@@ -148,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting