fluent-plugin-webhdfs 0.0.5 → 0.1.0

data/README.md

@@ -63,10 +63,33 @@ Store data as TSV (TAB separated values) of specified keys, without time, with t
 
 If message doesn't have specified attribute, fluent-plugin-webhdfs outputs 'NULL' instead of values.
 
+### Performance notifications
+
+Writing data on HDFS single file from 2 or more fluentd nodes, makes many bad blocks of HDFS. If you want to run 2 or more fluentd nodes with fluent-plugin-webhdfs, you should configure 'path' for each node.
+You can use '${hostname}' or '${uuid:random}' placeholders in configuration for this purpose.
+
+For hostname:
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /log/access/%Y%m%d/${hostname}.log
+    </match>
+
+Or with random filename (to avoid duplicated file name only):
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /log/access/%Y%m%d/${uuid:random}.log
+    </match>
+
+With configurations above, you can handle all of files of '/log/access/20120820/*' as specified timeslice access logs.
+
 ## TODO
 
-* long run test
-  * over webhdfs and httpfs
 * patches welcome!
 
 ## Copyright

data/fluent-plugin-webhdfs.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.0.5"
+  gem.version       = "0.1.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -16,8 +16,10 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rake"
   gem.add_development_dependency "fluentd"
   gem.add_development_dependency "fluent-mixin-plaintextformatter"
+  gem.add_development_dependency "fluent-mixin-config-placeholders"
   gem.add_development_dependency "webhdfs", '>= 0.5.1'
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter"
+  gem.add_runtime_dependency "fluent-mixin-config-placeholders"
   gem.add_runtime_dependency "webhdfs", '>= 0.5.1'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+require 'fluent/mixin/config_placeholders'
 require 'fluent/mixin/plaintextformatter'
 
 class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
@@ -12,6 +13,8 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   config_param :port, :integer, :default => 50070
   config_param :namenode, :string, :default => nil # host:port
 
+  include Fluent::Mixin::ConfigPlaceholders
+
   config_param :path, :string
   config_param :username, :string, :default => nil
 

data/test/plugin/test_out_webhdfs.rb

@@ -41,6 +41,15 @@ username hdfs_user
     assert_equal 'hdfs_user', d.instance.username
   end
 
+  def test_configure_placeholders
+    d = create_driver %[
+hostname testing.node.local
+namenode server.local:50070
+path /hdfs/${hostname}/file.%Y%m%d%H.log
+]
+    assert_equal '/hdfs/testing.node.local/file.%Y%m%d%H.log', d.instance.path
+  end
+
   def test_path_format
     d = create_driver
     assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-15 00:00:00.000000000 Z
+date: 2012-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -59,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +123,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement