fluent-plugin-webhdfs 0.1.1 → 0.1.3

data/README.md

@@ -5,6 +5,7 @@ Fluentd output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
 WebHDFSOutput slices data by time (specified unit), and store these data as hdfs file of plain text. You can specify to:
 
 * format whole data as serialized JSON, single attribute or separated multi attributes
+  * or LTSV, labeled-TSV (see http://ltsv.org/ )
 * include time as line header, or not
 * include tag as line header, or not
 * change field separator (default: TAB)
@@ -25,6 +26,16 @@ To store data by time,tag,json (same with 'type file') over WebHDFS:
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
     </match>
 
+To store data as LTSV without time and tag over WebHDFS:
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+      output_data_type ltsv
+    </match>
+
 With username of pseudo authentication:
 
     <match access.**>
@@ -88,6 +99,20 @@ Or with random filename (to avoid duplicated file name only):
 
 With configurations above, you can handle all of files of '/log/access/20120820/*' as specified timeslice access logs.
 
+### For unstable Namenodes
+
+With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inacive NameNodes.
+
+If you were usging unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /log/access/%Y%m%d/${hostname}.log
+      ignore_start_check_error true
+    </match>
+
 ## TODO
 
 * patches welcome!

data/fluent-plugin-webhdfs.gemspec

@@ -1,7 +1,8 @@
 # -*- encoding: utf-8 -*-
+
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.1.1"
+  gem.version       = "0.1.3"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -15,11 +16,11 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "fluentd"
-  gem.add_development_dependency "fluent-mixin-plaintextformatter", '>= 0.2.0'
-  gem.add_development_dependency "fluent-mixin-config-placeholders"
+  gem.add_development_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
+  gem.add_development_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
   gem.add_development_dependency "webhdfs", '>= 0.5.1'
   gem.add_runtime_dependency "fluentd"
-  gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.0'
-  gem.add_runtime_dependency "fluent-mixin-config-placeholders"
+  gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
+  gem.add_runtime_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
   gem.add_runtime_dependency "webhdfs", '>= 0.5.1'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -13,6 +13,8 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   config_param :port, :integer, :default => 50070
   config_param :namenode, :string, :default => nil # host:port
 
+  config_param :ignore_start_check_error, :bool, :default => false
+
   include Fluent::Mixin::ConfigPlaceholders
 
   config_param :path, :string
@@ -74,15 +76,13 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   def start
     super
 
-    noerror = false
     begin
       ary = @client.list('/')
-      noerror = true
+      $log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
     rescue
       $log.error "webdhfs check request failed!"
-      raise
+      raise unless @ignore_start_check_error
     end
-    $log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
   end
 
   def shutdown

data/test/plugin/test_out_webhdfs.rb

@@ -18,6 +18,7 @@ path /hdfs/path/file.%Y%m%d.log
     assert_equal '%Y%m%d', d.instance.time_slice_format
     assert_equal false, d.instance.httpfs
     assert_nil d.instance.username
+    assert_equal false, d.instance.ignore_start_check_error
     
     assert_equal true, d.instance.output_include_time
     assert_equal true, d.instance.output_include_tag

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-06 00:00:00.000000000 Z
+date: 2013-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.2.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,23 +58,23 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,23 +122,23 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -183,15 +183,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -2593820620278035456
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -2593820620278035456
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting