fluent-plugin-webhdfs 0.0.3 → 0.0.4

data/README.md

@@ -1,29 +1,76 @@
-# Fluent::Plugin::Webhdfs
+# fluent-plugin-webhdfs
 
-TODO: Write a gem description
+Fluentd output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
 
-## Installation
+WebHDFSOutput slices data by time (specified unit), and store these data as hdfs file of plain text. You can specify to:
 
-Add this line to your application's Gemfile:
+* format whole data as serialized JSON, single attribute or separated multi attributes
+* include time as line header, or not
+* include tag as line header, or not
+* change field separator (default: TAB)
+* add new line as termination, or not
 
-    gem 'fluent-plugin-webhdfs'
+And you can specify output file path as 'path /path/to/dir/access.%Y%m%d.log', then got '/path/to/dir/access.20120316.log' on HDFS.
 
-And then execute:
+## Configuration
 
-    $ bundle
+### WebHDFSOutput
 
-Or install it yourself as:
+To store data by time,tag,json (same with 'type file') over WebHDFS:
 
-    $ gem install fluent-plugin-webhdfs
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+    </match>
 
-## Usage
+With username of pseudo authentication:
 
-TODO: Write usage instructions here
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+      username hdfsuser
+    </match>
+      
+Store data over HttpFs (instead of WebHDFS):
 
-## Contributing
+    <match access.**>
+      type webhdfs
+      host httpfs.node.your.cluster.local
+      port 14000
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+      httpfs true
+    </match>
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+Store data as TSV (TAB separated values) of specified keys, without time, with tag (removed prefix 'access'):
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+
+      field_separator TAB        # or 'SPACE', 'COMMA'
+      output_include_time false
+      output_include_tag true
+      remove_prefix access
+
+      output_data_type attr:path,status,referer,agent,bytes
+    </match>
+
+If message doesn't have specified attribute, fluent-plugin-webhdfs outputs 'NULL' instead of values.
+
+## TODO
+
+* long run test
+  * over webhdfs and httpfs
+* patches welcome!
+
+## Copyright
+
+* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -1,2 +1,11 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-webhdfs.gemspec

@@ -1,12 +1,12 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.0.3"
+  gem.version       = "0.0.4"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
-  gem.description   = %q{For WebHDFS (and HttpFs experimentally)}
-  gem.homepage      = "https://github.com/tagomoris/fluent-plugin-webhdfs"
+  gem.description   = %q{For WebHDFS and HttpFs of Hadoop HDFS}
+  gem.homepage      = "https://github.com/fluent/fluent-plugin-webhdfs"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/fluent/plugin/out_webhdfs.rb

@@ -8,7 +8,10 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   config_set_default :buffer_type, 'memory'
   config_set_default :time_slice_format, '%Y%m%d'
 
-  config_param :namenode, :string # host:port
+  config_param :host, :string, :default => nil
+  config_param :port, :integer, :default => 50070
+  config_param :namenode, :string, :default => nil # host:port
+
   config_param :path, :string
   config_param :username, :string, :default => nil
 
@@ -16,6 +19,8 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
   include Fluent::Mixin::PlainTextFormatter
 
+  config_param :default_tag, :string, :default => 'tag_missing'
+
   def initialize
     super
     require 'net/http'
@@ -36,11 +41,18 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
     super
 
-    unless /\A([a-zA-Z0-9][-a-zA-Z0-9.]*):(\d+)\Z/ =~ @namenode
-      raise Fluent::ConfigError, "Invalid config value about namenode: '#{@namenode}', needs NAMENODE_NAME:PORT"
+    if @host
+      @namenode_host = @host
+      @namenode_port = @port
+    elsif @namenode
+      unless /\A([a-zA-Z0-9][-a-zA-Z0-9.]*):(\d+)\Z/ =~ @namenode
+        raise Fluent::ConfigError, "Invalid config value about namenode: '#{@namenode}', needs NAMENODE_NAME:PORT"
+      end
+      @namenode_host = $1
+      @namenode_port = $2.to_i
+    else
+      raise Fluent::ConfigError, "WebHDFS host or namenode missing"
     end
-    @namenode_host = $1
-    @namenode_port = $2.to_i
     unless @path.index('/') == 0
       raise Fluent::ConfigError, "Path on hdfs MUST starts with '/', but '#{@path}'"
     end
@@ -52,7 +64,6 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     if @httpfs
       @client.httpfs_mode = true
     end
-    @mutex = Mutex.new
   end
 
   def start
@@ -73,13 +84,6 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     super
   end
 
-  def record_to_string(record)
-    record.to_json
-  end
-
-  # def format(tag, time, record)
-  # end
-
   def path_format(chunk_key)
     Time.strptime(chunk_key, @time_slice_format).strftime(@path)
   end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_webhdfs'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_webhdfs.rb

@@ -0,0 +1,58 @@
+require 'helper'
+
+class WebHDFSOutputTest < Test::Unit::TestCase
+  CONFIG = %[
+host namenode.local
+path /hdfs/path/file.%Y%m%d.log
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
+    assert_equal 50070, d.instance.instance_eval{ @namenode_port }
+    assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+    assert_equal '%Y%m%d', d.instance.time_slice_format
+    assert_equal false, d.instance.httpfs
+    assert_nil d.instance.username
+    
+    assert_equal true, d.instance.output_include_time
+    assert_equal true, d.instance.output_include_tag
+    assert_equal 'json', d.instance.output_data_type
+    assert_nil d.instance.remove_prefix
+    assert_equal 'TAB', d.instance.field_separator
+    assert_equal true, d.instance.add_newline
+    assert_equal 'tag_missing', d.instance.default_tag
+
+    d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+httpfs yes
+username hdfs_user
+]
+    assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
+    assert_equal 14000, d.instance.instance_eval{ @namenode_port }
+    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+    assert_equal true, d.instance.httpfs
+    assert_equal 'hdfs_user', d.instance.username
+  end
+
+  def test_path_format
+    d = create_driver
+    assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+    assert_equal '%Y%m%d', d.instance.time_slice_format
+    assert_equal '/hdfs/path/file.20120718.log', d.instance.path_format('20120718')
+
+    d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+]
+    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+    assert_equal '/hdfs/path/file.20120718.1503.log', d.instance.path_format('201207181503')
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-13 00:00:00.000000000 Z
+date: 2012-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -123,7 +123,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.5.0
-description: For WebHDFS (and HttpFs experimentally)
+description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
 executables: []
@@ -137,7 +137,9 @@ files:
 - Rakefile
 - fluent-plugin-webhdfs.gemspec
 - lib/fluent/plugin/out_webhdfs.rb
-homepage: https://github.com/tagomoris/fluent-plugin-webhdfs
+- test/helper.rb
+- test/plugin/test_out_webhdfs.rb
+homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -161,4 +163,6 @@ rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_out_webhdfs.rb