fluent-plugin-hoop 0.1.3 → 0.1.4

data/Gemfile

@@ -7,10 +7,10 @@ source "http://rubygems.org"
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
+  gem "bundler"
   gem "jeweler", "~> 1.6.4"
-  gem "rcov", ">= 0"
 end
 
 gem "rdoc"
-
+gem "fluentd"
+gem "fluent-mixin-plaintextformatter"

data/Rakefile

@@ -43,14 +43,6 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
-end
-
 task :default => :test
 
 require 'rdoc/task'

data/VERSION

@@ -1 +1 @@
-0.1.3
+0.1.4

data/fluent-plugin-hoop.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-hoop"
-  s.version = "0.1.3"
+  s.version = "0.1.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["TAGOMORI Satoshi"]
-  s.date = "2012-06-13"
+  s.date = "2012-07-13"
   s.description = "This plugin doesn't support Apache Hadoop's HttpFs. See fluent-plugin-webhdfs."
   s.email = "tagomoris@gmail.com"
   s.extra_rdoc_files = [
@@ -44,29 +44,32 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<rdoc>, [">= 0"])
+      s.add_runtime_dependency(%q<fluentd>, [">= 0"])
+      s.add_runtime_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.8"])
       s.add_development_dependency(%q<rake>, [">= 0.9.2"])
       s.add_development_dependency(%q<simplecov>, [">= 0.5.4"])
     else
       s.add_dependency(%q<rdoc>, [">= 0"])
+      s.add_dependency(%q<fluentd>, [">= 0"])
+      s.add_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
       s.add_dependency(%q<rake>, [">= 0.9.2"])
       s.add_dependency(%q<simplecov>, [">= 0.5.4"])
     end
   else
     s.add_dependency(%q<rdoc>, [">= 0"])
+    s.add_dependency(%q<fluentd>, [">= 0"])
+    s.add_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
     s.add_dependency(%q<rake>, [">= 0.9.2"])
     s.add_dependency(%q<simplecov>, [">= 0.5.4"])

data/lib/fluent/plugin/out_hoop.rb

@@ -1,224 +1,16 @@
-module FluentExt; end
-module FluentExt::PlainTextFormatterMixin
-  # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
-
-  attr_accessor :output_include_time, :output_include_tag, :output_data_type
-  attr_accessor :add_newline, :field_separator
-  attr_accessor :remove_prefix, :default_tag
-  
-  def configure(conf)
-    super
-
-    @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
-    @output_include_time = true if @output_include_time.nil?
-
-    @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
-    @output_include_tag = true if @output_include_tag.nil?
-
-    @output_data_type = conf['output_data_type']
-    @output_data_type = 'json' if @output_data_type.nil?
-
-    @field_separator = case conf['field_separator']
-                       when 'SPACE' then ' '
-                       when 'COMMA' then ','
-                       else "\t"
-                       end
-    @add_newline = Fluent::Config.bool_value(conf['add_newline'])
-    if @add_newline.nil?
-      @add_newline = true
-    end
-
-    @remove_prefix = conf['remove_prefix']
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
-    end
-    if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
-      @default_tag = conf['default_tag']
-      if @default_tag.nil? or @default_tag.length < 1
-        raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
-      end
-    end
-
-    # default timezone: utc
-    if conf['localtime'].nil? and conf['utc'].nil?
-      @utc = true
-      @localtime = false
-    elsif not @localtime and not @utc
-      @utc = true
-      @localtime = false
-    end
-    # mix-in default time formatter (or you can overwrite @timef on your own configure)
-    @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
-
-    @custom_attributes = []
-    if @output_data_type == 'json'
-      self.instance_eval {
-        def stringify_record(record)
-          record.to_json
-        end
-      }
-    elsif @output_data_type =~ /^attr:(.*)$/
-      @custom_attributes = $1.split(',')
-      if @custom_attributes.size > 1
-        self.instance_eval {
-          def stringify_record(record)
-            @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@field_separator)
-          end
-        }
-      elsif @custom_attributes.size == 1
-        self.instance_eval {
-          def stringify_record(record)
-            (record[@custom_attributes[0]] || 'NULL').to_s
-          end
-        }
-      else
-        raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
-      end
-    else
-      raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
-    end
-
-    if @output_include_time and @output_include_tag
-      if @add_newline and @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @add_newline
-        self.instance_eval {
-          def format(tag,time,record)
-            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record)
-            @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
-          end
-        }
-      end
-    elsif @output_include_time
-      if @add_newline
-        self.instance_eval {
-          def format(tag,time,record);
-            @timef.format(time) + @field_separator + stringify_record(record) + "\n"
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record);
-            @timef.format(time) + @field_separator + stringify_record(record)
-          end
-        }
-      end
-    elsif @output_include_tag
-      if @add_newline and @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            tag + @field_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @add_newline
-        self.instance_eval {
-          def format(tag,time,record)
-            tag + @field_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            tag + @field_separator + stringify_record(record)
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record)
-            tag + @field_separator + stringify_record(record)
-          end
-        }
-      end
-    else # without time, tag
-      if @add_newline
-        self.instance_eval {
-          def format(tag,time,record);
-            stringify_record(record) + "\n"
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record);
-            stringify_record(record)
-          end
-        }
-      end
-    end
-  end
-
-  def stringify_record(record)
-    record.to_json
-  end
-
-  def format(tag, time, record)
-    if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
-      tag = tag[@removed_length..-1] || @default_tag
-    end
-    time_str = if @output_include_time
-                 @timef.format(time) + @field_separator
-               else
-                 ''
-               end
-    tag_str = if @output_include_tag
-                tag + @field_separator
-              else
-                ''
-              end
-    time_str + tag_str + stringify_record(record) + "\n"
-  end
-
-end
+require 'fluent/mixin/plaintextformatter'
 
 class Fluent::HoopOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('hoop', self)
 
   config_set_default :buffer_type, 'memory'
   config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
-  # config_param :tag_format, :string, :default => 'all' # or 'last'(last.part.of.tag => tag) or 'none'
 
   config_param :hoop_server, :string   # host:port
   config_param :path, :string          # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S and %T(tag, not-supported-yet)
   config_param :username, :string      # hoop pseudo username
   
-  include FluentExt::PlainTextFormatterMixin
-  config_set_default :output_include_time, true
-  config_set_default :output_include_tag, true
-  config_set_default :output_data_type, 'json'
-  config_set_default :field_separator, "\t"
-  config_set_default :add_newline, true
-  config_set_default :remove_prefix, nil
+  include Fluent::Mixin::PlainTextFormatter
 
   def initialize
     super
@@ -249,12 +41,6 @@ class Fluent::HoopOutput < Fluent::TimeSlicedOutput
     end
     @conn = nil
     @header = {'Content-Type' => 'application/octet-stream'}
-
-    @f_separator = case @field_separator
-                   when 'SPACE' then ' '
-                   when 'COMMA' then ','
-                   else "\t"
-                   end
   end
 
   def start
@@ -286,10 +72,8 @@ class Fluent::HoopOutput < Fluent::TimeSlicedOutput
     record.to_json
   end
 
-  def format(tag, time, record)
-    time_str = @timef.format(time)
-    time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
-  end
+  # def format(tag, time, record)
+  # end
 
   def path_format(chunk_key)
     Time.strptime(chunk_key, @time_slice_format).strftime(@path)

data/test/plugin/test_out_hoop.rb

@@ -80,7 +80,7 @@ class HoopOutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.output_include_tag
     assert_equal 'json', d.instance.output_data_type
     assert_equal true, d.instance.add_newline
-    assert_equal "\t", d.instance.field_separator
+    assert_equal "TAB", d.instance.field_separator
 
     assert_nil d.instance.remove_prefix
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-hoop
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-13 00:00:00.000000000 Z
+date: 2012-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -28,14 +28,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: shoulda
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -44,39 +44,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: fluent-mixin-plaintextformatter
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: jeweler
+  name: shoulda
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.6.4
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.6.4
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rcov
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -91,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -176,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -355817273147021109
+      hash: 3002117588633528194
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: