fluent-plugin-webhdfs 0.0.2 → 0.0.3

data/fluent-plugin-webhdfs.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.0.2"
+  gem.version       = "0.0.3"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -15,7 +15,9 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "fluent-mixin-plaintextformatter"
   gem.add_development_dependency "webhdfs", '>= 0.5.0'
   gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "fluent-mixin-plaintextformatter"
   gem.add_runtime_dependency "webhdfs", '>= 0.5.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -1,12 +1,10 @@
 # -*- coding: utf-8 -*-
 
-require_relative 'ext_mixin'
+require 'fluent/mixin/plaintextformatter'
 
 class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('webhdfs', self)
 
-  WEBHDFS_VERSION = 'v1'
-
   config_set_default :buffer_type, 'memory'
   config_set_default :time_slice_format, '%Y%m%d'
 
@@ -16,13 +14,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
   config_param :httpfs, :bool, :default => false
 
-  include FluentExt::PlainTextFormatterMixin
-  config_set_default :output_include_time, true
-  config_set_default :output_include_tag, true
-  config_set_default :output_data_type, 'json'
-  config_set_default :field_separator, "\t"
-  config_set_default :add_newline, true
-  config_set_default :remove_prefix, nil
+  include Fluent::Mixin::PlainTextFormatter
 
   def initialize
     super
@@ -54,12 +46,6 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
     @conn = nil
     
-    @f_separator = case @field_separator
-                   when 'SPACE' then ' '
-                   when 'COMMA' then ','
-                   else "\t"
-                   end
-
     # path => cached_url
     # @cached_datanode_urls = {}
     @client = WebHDFS::Client.new(@namenode_host, @namenode_port, @username)
@@ -91,10 +77,8 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     record.to_json
   end
 
-  def format(tag, time, record)
-    time_str = @timef.format(time)
-    time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
-  end
+  # def format(tag, time, record)
+  # end
 
   def path_format(chunk_key)
     Time.strptime(chunk_key, @time_slice_format).strftime(@path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-18 00:00:00.000000000 Z
+date: 2012-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-plaintextformatter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -75,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-plaintextformatter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -104,7 +136,6 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-webhdfs.gemspec
-- lib/fluent/plugin/ext_mixin.rb
 - lib/fluent/plugin/out_webhdfs.rb
 homepage: https://github.com/tagomoris/fluent-plugin-webhdfs
 licenses: []

data/lib/fluent/plugin/ext_mixin.rb

@@ -1,207 +0,0 @@
-module FluentExt; end
-
-module FluentExt::PlainTextFormatterMixin
-  #TODO: tests!
-  
-  # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
-
-  attr_accessor :output_include_time, :output_include_tag, :output_data_type
-  attr_accessor :add_newline, :field_separator
-  attr_accessor :remove_prefix, :default_tag
-  
-  attr_accessor :f_separator
-
-  def configure(conf)
-    super
-
-    @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
-    @output_include_time = true if @output_include_time.nil?
-
-    @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
-    @output_include_tag = true if @output_include_tag.nil?
-
-    @output_data_type = conf['output_data_type']
-    @output_data_type = 'json' if @output_data_type.nil?
-
-    @f_separator = case conf['field_separator']
-                   when 'SPACE' then ' '
-                   when 'COMMA' then ','
-                   else "\t"
-                   end
-    @add_newline = Fluent::Config.bool_value(conf['add_newline'])
-    if @add_newline.nil?
-      @add_newline = true
-    end
-
-    @remove_prefix = conf['remove_prefix']
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
-    end
-    if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
-      @default_tag = conf['default_tag']
-      if @default_tag.nil? or @default_tag.length < 1
-        raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
-      end
-    end
-
-    # default timezone: utc
-    if conf['localtime'].nil? and conf['utc'].nil?
-      @utc = true
-      @localtime = false
-    elsif not @localtime and not @utc
-      @utc = true
-      @localtime = false
-    end
-    # mix-in default time formatter (or you can overwrite @timef on your own configure)
-    @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
-
-    @custom_attributes = []
-    if @output_data_type == 'json'
-      self.instance_eval {
-        def stringify_record(record)
-          record.to_json
-        end
-      }
-    elsif @output_data_type =~ /^attr:(.*)$/
-      @custom_attributes = $1.split(',')
-      if @custom_attributes.size > 1
-        self.instance_eval {
-          def stringify_record(record)
-            @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@f_separator)
-          end
-        }
-      elsif @custom_attributes.size == 1
-        self.instance_eval {
-          def stringify_record(record)
-            (record[@custom_attributes[0]] || 'NULL').to_s
-          end
-        }
-      else
-        raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
-      end
-    else
-      raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
-    end
-
-    if @output_include_time and @output_include_tag
-      if @add_newline and @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @add_newline
-        self.instance_eval {
-          def format(tag,time,record)
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record)
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
-          end
-        }
-      end
-    elsif @output_include_time
-      if @add_newline
-        self.instance_eval {
-          def format(tag,time,record);
-            @timef.format(time) + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record);
-            @timef.format(time) + @f_separator + stringify_record(record)
-          end
-        }
-      end
-    elsif @output_include_tag
-      if @add_newline and @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @add_newline
-        self.instance_eval {
-          def format(tag,time,record)
-            tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            tag + @f_separator + stringify_record(record)
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record)
-            tag + @f_separator + stringify_record(record)
-          end
-        }
-      end
-    else # without time, tag
-      if @add_newline
-        self.instance_eval {
-          def format(tag,time,record);
-            stringify_record(record) + "\n"
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record);
-            stringify_record(record)
-          end
-        }
-      end
-    end
-  end
-
-  def stringify_record(record)
-    record.to_json
-  end
-
-  def format(tag, time, record)
-    if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
-      tag = tag[@removed_length..-1] || @default_tag
-    end
-    time_str = if @output_include_time
-                 @timef.format(time) + @f_separator
-               else
-                 ''
-               end
-    tag_str = if @output_include_tag
-                tag + @f_separator
-              else
-                ''
-              end
-    time_str + tag_str + stringify_record(record) + "\n"
-  end
-
-end