fluent-plugin-webhdfs 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56b8e7efe377002f3f092f79152826975098b9f8
-  data.tar.gz: 77ef93b880901d5136980b63125f0a36b620fd9c
+  metadata.gz: 08f6f306b0cc2e49582bc066e03d3460c69fd245
+  data.tar.gz: deb9774da6e4d8da83a844603254e4e3a069d78d
 SHA512:
-  metadata.gz: d8f2660f0c17f7956fba570deea3ed4a3301e8ea54f9b3f0e67232c5f24ba508949cfd259398eb22d4a20be1d4a83e1637ddf68c730919e619872b2382d5eac2
-  data.tar.gz: 3d834d669fab4f4986e46a9e439afe983b53264dd7555db1c48c3565ac4a3cbf82b3d5ebb5c90d0259edde0463b0cfadffde2b93f8a9020e877fe9160e3cc52b
+  metadata.gz: 6dc0afebf5e982bcdfa12ae746f17925e4ddc4889a8d0fa4087d3d4d866889f9b3879b75b02a03ebcd833ce39d3815e614da2fc19305e14163101e7d3236c9e6
+  data.tar.gz: ddd01525b9bc7536eed1a36055298cddee927b7a8be8b1feb5ac913e07766faad050d2ff8c2d2203a12ad4a1ec79e587893f7b03faf989b4cbf7c9a07ab7576c

data/.travis.yml

@@ -4,7 +4,6 @@ rvm:
   - 1.9.2
   - 1.9.3
   - 2.0.0
-  - rbx-19mode
 
 branches:
   only:

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-webhdfs
 
-Fluentd output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
+[Fluentd](http://fluentd.org/) output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
 
 WebHDFSOutput slices data by time (specified unit), and store these data as hdfs file of plain text. You can specify to:
 
@@ -26,6 +26,17 @@ To store data by time,tag,json (same with 'type file') over WebHDFS:
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
     </match>
 
+If you want JSON object only (without time or tag or both on header of lines), specify it by `output_include_time` or `output_include_tag` (default true):
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+      output_include_time false
+      output_include_tag  false
+    </match>
+
 To specify namenode, `namenode` is also available:
 
     <match access.**>

data/fluent-plugin-webhdfs.gemspec

@@ -2,12 +2,13 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.2.1"
+  gem.version       = "0.2.2"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
   gem.description   = %q{For WebHDFS and HttpFs of Hadoop HDFS}
   gem.homepage      = "https://github.com/fluent/fluent-plugin-webhdfs"
+  gem.license       = "APLv2"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/fluent/plugin/out_webhdfs.rb

@@ -50,6 +50,11 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     require 'webhdfs'
   end
 
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   def configure(conf)
     if conf['path']
       if conf['path'].index('%S')
@@ -125,7 +130,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
       begin
         client.list('/')
       rescue => e
-        $log.warn "webhdfs check request failed. (namenode: #{client.host}:#{client.port}, error: #{e.message})"
+        log.warn "webhdfs check request failed. (namenode: #{client.host}:#{client.port}, error: #{e.message})"
         available = false
       end
       available
@@ -138,11 +143,11 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     super
 
     if namenode_available(@client)
-      $log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
+      log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
       return
     end
     if @client_standby && namenode_available(@client_standby)
-      $log.info "webhdfs connection confirmed: #{@standby_namenode_host}:#{@standby_namenode_port}"
+      log.info "webhdfs connection confirmed: #{@standby_namenode_host}:#{@standby_namenode_port}"
       return
     end
 
@@ -166,7 +171,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   def namenode_failover
     if @standby_namenode
       @client, @client_standby = @client_standby, @client
-      $log.warn "Namenode failovered, now using #{@client.host}:#{@client.port}."
+      log.warn "Namenode failovered, now using #{@client.host}:#{@client.port}."
     end
   end
 
@@ -199,18 +204,18 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     begin
       send_data(hdfs_path, chunk.read)
     rescue => e
-      $log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
+      log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
 
       raise e if !@client_standby || failovered
 
       if is_standby_exception(e) && namenode_available(@client_standby)
-        $log.warn "Seems the connected host status is not active (maybe due to failovers). Gonna try another namenode immediately."
+        log.warn "Seems the connected host status is not active (maybe due to failovers). Gonna try another namenode immediately."
         namenode_failover
         failovered = true
         retry
       end
       if ((@error_history.size + 1) >= @failures_before_use_standby) && namenode_available(@client_standby)
-        $log.warn "Too many failures. Try to use the standby namenode instead."
+        log.warn "Too many failures. Try to use the standby namenode instead."
         namenode_failover
         failovered = true
         retry

metadata

@@ -1,83 +1,83 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-23 00:00:00.000000000 Z
+date: 2014-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-plaintextformatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.3
 description: For WebHDFS and HttpFs of Hadoop HDFS
@@ -87,8 +87,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -98,7 +98,8 @@ files:
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
-licenses: []
+licenses:
+- APLv2
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -106,21 +107,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
 test_files:
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
-has_rdoc: