fluent-plugin-webhdfs 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65580c8eb45e57136473c4713aef1b7bf8b7face
-  data.tar.gz: 75867f172f39ffdf0a1d3f9c94eeb703ad062b97
+  metadata.gz: b109af994a373ce05a964a0b1560bfad64b08f6d
+  data.tar.gz: 3bf1cc0de4b54d399158ead361f90825d4f8ba92
 SHA512:
-  metadata.gz: 27acabdcaf0716e4daa491f208ca29d4441e0b9e2bc4b270d2652de65695f3e24dae0a938d8b5d7cedc186bc7137569ab3a81878c51fe3553814a81a5b98ca88
-  data.tar.gz: 2ad981a4bbf498be9b828e54202d0ecea87b0646f69605068281032ca4f9c891e7251967b8b70bbc5d1c50fc34cf7f65b20c52843c33c60f785dcea8b6365f1b
+  metadata.gz: e1ca791673549cc32e874360d07c57005af2a2c88e1d7b3ad6a5bd491cfa01d1724f196a793b8294eca3b8448792d3c2b148006ac61ad24b8e583050bcd2a789
+  data.tar.gz: ad02485500aa4af1b1f8bff78e91722ebebad6387bbbe4c53e89ba7c094119693afe8ff0fa42018f9a75ac206d689c1b5fcae8b6e215305d1a92db22f476c23d

data/README.md

@@ -26,6 +26,14 @@ To store data by time,tag,json (same with 'type file') over WebHDFS:
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
     </match>
 
+To specify namenode, `namenode` is also available:
+
+    <match access.**>
+      type     webhdfs
+      namenode master.your.cluster.local:50070
+      path     /path/on/hdfs/access.log.%Y%m%d_%H.log
+    </match>
+
 To store data as LTSV without time and tag over WebHDFS:
 
     <match access.**>
@@ -74,6 +82,28 @@ Store data as TSV (TAB separated values) of specified keys, without time, with t
 
 If message doesn't have specified attribute, fluent-plugin-webhdfs outputs 'NULL' instead of values.
 
+### Namenode HA / Auto retry for WebHDFS known errors
+
+`fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:
+
+    <match access.**>
+      type             webhdfs
+      namenode         master1.your.cluster.local:50070
+	  standby_namenode master2.your.cluster.local:50070
+      path             /path/on/hdfs/access.log.%Y%m%d_%H.log
+    </match>
+
+And you can also specify to retry known hdfs errors (such like `LeaseExpiredException`) automatically. With this configuration, fluentd doesn't write logs for this errors if retry successed.
+
+    <match access.**>
+      type               webhdfs
+      namenode           master1.your.cluster.local:50070
+      path               /path/on/hdfs/access.log.%Y%m%d_%H.log
+	  retry_known_errors yes
+	  retry_times        1 # default 1
+	  retry_interval     1 # [sec] default 1
+    </match>
+
 ### Performance notifications
 
 Writing data on HDFS single file from 2 or more fluentd nodes, makes many bad blocks of HDFS. If you want to run 2 or more fluentd nodes with fluent-plugin-webhdfs, you should configure 'path' for each node.
@@ -99,6 +129,16 @@ Or with random filename (to avoid duplicated file name only):
 
 With configurations above, you can handle all of files of '/log/access/20120820/*' as specified timeslice access logs.
 
+For high load cluster nodes, you can specify timeouts for HTTP requests.
+
+    <match access.**>
+	  type webhdfs
+	  namenode master.your.cluster.local:50070
+      path /log/access/%Y%m%d/${hostname}.log
+	  open_timeout 180 # [sec] default: 30
+	  read_timeout 180 # [sec] default: 60
+    </match>
+
 ### For unstable Namenodes
 
 With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inacive NameNodes.
@@ -115,6 +155,8 @@ If you were usging unstable NameNodes and have wanted to ignore NameNode errors
 
 ## TODO
 
+* configuration example for Hadoop Namenode HA
+  * here, or docs.fluentd.org ?
 * patches welcome!
 
 ## Copyright

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.1.4"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -18,5 +18,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
-  gem.add_runtime_dependency "webhdfs", '>= 0.5.1'
+  gem.add_runtime_dependency "webhdfs", '>= 0.5.3'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -12,6 +12,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   config_param :host, :string, :default => nil
   config_param :port, :integer, :default => 50070
   config_param :namenode, :string, :default => nil # host:port
+  config_param :standby_namenode, :string, :default => nil # host:port
 
   config_param :ignore_start_check_error, :bool, :default => false
 
@@ -25,6 +26,15 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   config_param :open_timeout, :integer, :default => 30 # from ruby net/http default
   config_param :read_timeout, :integer, :default => 60 # from ruby net/http default
 
+  config_param :retry_known_errors, :bool, :default => false
+  config_param :retry_interval, :integer, :default => nil
+  config_param :retry_times, :integer, :default => nil
+
+  # how many times of write failure before switch to standby namenode
+  # by default it's 11 times that costs 1023 seconds inside fluentd,
+  # which is considered enough to exclude the scenes that caused by temporary network fail or single datanode fail
+  config_param :failures_before_use_standby, :integer, :default => 11
+
   include Fluent::Mixin::PlainTextFormatter
 
   config_param :default_tag, :string, :default => 'tag_missing'
@@ -54,34 +64,80 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
       @namenode_port = @port
     elsif @namenode
       unless /\A([a-zA-Z0-9][-a-zA-Z0-9.]*):(\d+)\Z/ =~ @namenode
-        raise Fluent::ConfigError, "Invalid config value about namenode: '#{@namenode}', needs NAMENODE_NAME:PORT"
+        raise Fluent::ConfigError, "Invalid config value about namenode: '#{@namenode}', needs NAMENODE_HOST:PORT"
       end
       @namenode_host = $1
       @namenode_port = $2.to_i
     else
       raise Fluent::ConfigError, "WebHDFS host or namenode missing"
     end
+    if @standby_namenode
+      unless /\A([a-zA-Z0-9][-a-zA-Z0-9.]*):(\d+)\Z/ =~ @standby_namenode
+        raise Fluent::ConfigError, "Invalid config value about standby namenode: '#{@standby_namenode}', needs STANDBY_NAMENODE_HOST:PORT"
+      end
+      if @httpfs
+        raise Fluent::ConfigError, "Invalid configuration: specified to use both of standby_namenode and httpfs."
+      end
+      @standby_namenode_host = $1
+      @standby_namenode_port = $2.to_i
+    end
     unless @path.index('/') == 0
       raise Fluent::ConfigError, "Path on hdfs MUST starts with '/', but '#{@path}'"
     end
-    
-    @client = WebHDFS::Client.new(@namenode_host, @namenode_port, @username)
+
+    @client = prepare_client(@namenode_host, @namenode_port, @username)
+    if @standby_namenode_host
+      @client_standby = prepare_client(@standby_namenode_host, @standby_namenode_port, @username)
+    else
+      @client_standby = nil
+    end
+  end
+
+  def prepare_client(host, port, username)
+    client = WebHDFS::Client.new(host, port, username)
     if @httpfs
-      @client.httpfs_mode = true
+      client.httpfs_mode = true
+    end
+    client.open_timeout = @open_timeout
+    client.read_timeout = @read_timeout
+    if @retry_known_errors
+      client.retry_known_errors = true
+      client.retry_interval = @retry_interval if @retry_interval
+      client.retry_times = @retry_times if @retry_times
+    end
+
+    client
+  end
+
+  def namenode_available(client)
+    if client
+      available = true
+      begin
+        client.list('/')
+      rescue => e
+        $log.warn "webhdfs check request failed. (namenode: #{client.host}:#{client.port}, error: #{e.message})"
+        available = false
+      end
+      available
+    else
+      false
     end
-    @client.open_timeout = @open_timeout
-    @client.read_timeout = @read_timeout
   end
 
   def start
     super
 
-    begin
-      ary = @client.list('/')
+    if namenode_available(@client)
       $log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
-    rescue
-      $log.error "webdhfs check request failed!"
-      raise unless @ignore_start_check_error
+      return
+    end
+    if @client_standby && namenode_available(@client_standby)
+      $log.info "webhdfs connection confirmed: #{@standby_namenode_host}:#{@standby_namenode_port}"
+      return
+    end
+
+    unless @ignore_start_check_error
+      raise RuntimeError, "webhdfs is not available now."
     end
   end
 
@@ -93,8 +149,19 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     Time.strptime(chunk_key, @time_slice_format).strftime(@path)
   end
 
+  def is_standby_exception(e)
+    e.is_a?(WebHDFS::IOError) && e.message.match(/org\.apache\.hadoop\.ipc\.StandbyException/)
+  end
+
+  def namenode_failover
+    if @standby_namenode
+      @client, @client_standby = @client_standby, @client
+      $log.warn "Namenode failovered, now using #{@client.host}:#{@client.port}."
+    end
+  end
+
   # TODO check conflictions
-  
+
   def send_data(path, data)
     begin
       @client.append(path, data)
@@ -105,11 +172,27 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
   def write(chunk)
     hdfs_path = path_format(chunk.key)
+    failovered = false
     begin
       send_data(hdfs_path, chunk.read)
-    rescue
-      $log.error "failed to communicate hdfs cluster, path: #{hdfs_path}"
-      raise
+    rescue => e
+      $log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
+
+      raise e if !@client_standby || failovered
+
+      if is_standby_exception(e) && namenode_available(@client_standby)
+        $log.warn "Seems the connected host status is not active (maybe due to failovers). Gonna try another namenode immediately."
+        namenode_failover
+        failovered = true
+        retry
+      end
+      if ((@error_history.size + 1) >= @failures_before_use_standby) && namenode_available(@client_standby)
+        $log.warn "Too many failures. Try to use the standby namenode instead."
+        namenode_failover
+        failovered = true
+        retry
+      end
+      raise e
     end
     hdfs_path
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-08 00:00:00.000000000 Z
+date: 2013-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.5.1
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.5.1
+        version: 0.5.3
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com