fluent-plugin-webhdfs 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b109af994a373ce05a964a0b1560bfad64b08f6d
-  data.tar.gz: 3bf1cc0de4b54d399158ead361f90825d4f8ba92
+  metadata.gz: 56b8e7efe377002f3f092f79152826975098b9f8
+  data.tar.gz: 77ef93b880901d5136980b63125f0a36b620fd9c
 SHA512:
-  metadata.gz: e1ca791673549cc32e874360d07c57005af2a2c88e1d7b3ad6a5bd491cfa01d1724f196a793b8294eca3b8448792d3c2b148006ac61ad24b8e583050bcd2a789
-  data.tar.gz: ad02485500aa4af1b1f8bff78e91722ebebad6387bbbe4c53e89ba7c094119693afe8ff0fa42018f9a75ac206d689c1b5fcae8b6e215305d1a92db22f476c23d
+  metadata.gz: d8f2660f0c17f7956fba570deea3ed4a3301e8ea54f9b3f0e67232c5f24ba508949cfd259398eb22d4a20be1d4a83e1637ddf68c730919e619872b2382d5eac2
+  data.tar.gz: 3d834d669fab4f4986e46a9e439afe983b53264dd7555db1c48c3565ac4a3cbf82b3d5ebb5c90d0259edde0463b0cfadffde2b93f8a9020e877fe9160e3cc52b

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - rbx-19mode
+
+branches:
+  only:
+    - master
+
+script: bundle exec rake test

data/README.md

@@ -153,6 +153,21 @@ If you were usging unstable NameNodes and have wanted to ignore NameNode errors
       ignore_start_check_error true
     </match>
 
+### For unstable Datanodes
+
+With unstable datanodes that frequently downs, appending over WebHDFS may produce broken files. In such cases, specify `append no` and `${chunk_id}` parameter.
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      
+      append no
+      path   /log/access/%Y%m%d/${hostname}.${chunk_id}.log
+    </match>
+
+`out_webhdfs` creates new files on hdfs per flush of fluentd, with chunk id. You shouldn't care broken files from append operations.
+
 ## TODO
 
 * configuration example for Hadoop Namenode HA

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.2.0"
+  gem.version       = "0.2.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}

data/lib/fluent/plugin/out_webhdfs.rb

@@ -39,6 +39,10 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
   config_param :default_tag, :string, :default => 'tag_missing'
 
+  config_param :append, :bool, :default => true
+
+  CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
+
   def initialize
     super
     require 'net/http'
@@ -91,6 +95,12 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     else
       @client_standby = nil
     end
+
+    if not @append
+      if @path.index(CHUNK_ID_PLACE_HOLDER).nil?
+        raise Fluent::ConfigError, "path must contain ${chunk_id}, which is the placeholder for chunk_id, when append is set to false."
+      end
+    end
   end
 
   def prepare_client(host, port, username)
@@ -160,18 +170,31 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
+  def chunk_unique_id_to_str(unique_id)
+    unique_id.unpack('C*').map{|x| x.to_s(16).rjust(2,'0')}.join('')
+  end
+
   # TODO check conflictions
 
   def send_data(path, data)
-    begin
-      @client.append(path, data)
-    rescue WebHDFS::FileNotFoundError
-      @client.create(path, data)
+    if @append
+      begin
+        @client.append(path, data)
+      rescue WebHDFS::FileNotFoundError
+        @client.create(path, data)
+      end
+    else
+      @client.create(path, data, {'overwrite' => 'true'})
     end
   end
 
   def write(chunk)
-    hdfs_path = path_format(chunk.key)
+    hdfs_path = if @append
+                  path_format(chunk.key)
+                else
+                  path_format(chunk.key).gsub(CHUNK_ID_PLACE_HOLDER, chunk_unique_id_to_str(chunk.unique_id))
+                end
+
     failovered = false
     begin
       send_data(hdfs_path, chunk.read)

data/test/plugin/test_out_webhdfs.rb

@@ -64,5 +64,13 @@ path /hdfs/path/file.%Y%m%d.%H%M.log
     assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
     assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
     assert_equal '/hdfs/path/file.20120718.1503.log', d.instance.path_format('201207181503')
+
+    assert_raise Fluent::ConfigError do
+      d = create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/file.%Y%m%d.%H%M.log
+            append false
+          ]
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-15 00:00:00.000000000 Z
+date: 2013-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -88,6 +88,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -115,10 +116,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.0.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
 test_files:
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
+has_rdoc: