fluent-plugin-file-alternative 0.1.2 → 0.1.3

data/README.md

@@ -2,7 +2,7 @@
 
 File output plugin alternative implementation, **is 100% compatible with fluentd built-in 'out_file'**, and added many options to format output as you want.
 
-FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on hdfs. You can specify to:
+FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on local file. You can specify to:
 
 * format whole data as serialized JSON, single attribute or separated multi attributes
 * include time as line header, or not

data/fluent-plugin-file-alternative.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-file-alternative"
-  gem.version       = "0.1.2"
+  gem.version       = "0.1.3"
 
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
@@ -16,5 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "fluent-mixin-plaintextformatter"
   gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "fluent-mixin-plaintextformatter"
 end

data/lib/fluent/plugin/out_file_alternative.rb

@@ -1,207 +1,4 @@
-module FluentExt; end
-module FluentExt::PlainTextFormatterMixin
-  # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
-
-  attr_accessor :output_include_time, :output_include_tag, :output_data_type
-  attr_accessor :add_newline, :field_separator
-  attr_accessor :remove_prefix, :default_tag
-  
-  attr_accessor :f_separator
-
-  def configure(conf)
-    super
-
-    @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
-    @output_include_time = true if @output_include_time.nil?
-
-    @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
-    @output_include_tag = true if @output_include_tag.nil?
-
-    @output_data_type = conf['output_data_type']
-    @output_data_type = 'json' if @output_data_type.nil?
-
-    @f_separator = case conf['field_separator']
-                   when 'SPACE' then ' '
-                   when 'COMMA' then ','
-                   else "\t"
-                   end
-    @add_newline = Fluent::Config.bool_value(conf['add_newline'])
-    if @add_newline.nil?
-      @add_newline = true
-    end
-
-    @remove_prefix = conf['remove_prefix']
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
-    end
-    if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
-      @default_tag = conf['default_tag']
-      if @default_tag.nil? or @default_tag.length < 1
-        raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
-      end
-    end
-
-    # default timezone: utc
-    if conf['localtime'].nil? and conf['utc'].nil?
-      @utc = true
-      @localtime = false
-    elsif not @localtime and not @utc
-      @utc = true
-      @localtime = false
-    end
-    # mix-in default time formatter (or you can overwrite @timef on your own configure)
-    @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
-
-    @custom_attributes = []
-    if @output_data_type == 'json'
-      self.instance_eval {
-        def stringify_record(record)
-          record.to_json
-        end
-      }
-    elsif @output_data_type =~ /^attr:(.*)$/
-      @custom_attributes = $1.split(',')
-      if @custom_attributes.size > 1
-        self.instance_eval {
-          def stringify_record(record)
-            @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@f_separator)
-          end
-        }
-      elsif @custom_attributes.size == 1
-        self.instance_eval {
-          def stringify_record(record)
-            (record[@custom_attributes[0]] || 'NULL').to_s
-          end
-        }
-      else
-        raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
-      end
-    else
-      raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
-    end
-
-    if @output_include_time and @output_include_tag
-      if @add_newline and @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @add_newline
-        self.instance_eval {
-          def format(tag,time,record)
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record)
-            @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
-          end
-        }
-      end
-    elsif @output_include_time
-      if @add_newline
-        self.instance_eval {
-          def format(tag,time,record);
-            @timef.format(time) + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record);
-            @timef.format(time) + @f_separator + stringify_record(record)
-          end
-        }
-      end
-    elsif @output_include_tag
-      if @add_newline and @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @add_newline
-        self.instance_eval {
-          def format(tag,time,record)
-            tag + @f_separator + stringify_record(record) + "\n"
-          end
-        }
-      elsif @remove_prefix
-        self.instance_eval {
-          def format(tag,time,record)
-            if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-                tag == @remove_prefix
-              tag = tag[@removed_length..-1] || @default_tag
-            end
-            tag + @f_separator + stringify_record(record)
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record)
-            tag + @f_separator + stringify_record(record)
-          end
-        }
-      end
-    else # without time, tag
-      if @add_newline
-        self.instance_eval {
-          def format(tag,time,record);
-            stringify_record(record) + "\n"
-          end
-        }
-      else
-        self.instance_eval {
-          def format(tag,time,record);
-            stringify_record(record)
-          end
-        }
-      end
-    end
-  end
-
-  def stringify_record(record)
-    record.to_json
-  end
-
-  def format(tag, time, record)
-    if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
-      tag = tag[@removed_length..-1] || @default_tag
-    end
-    time_str = if @output_include_time
-                 @timef.format(time) + @f_separator
-               else
-                 ''
-               end
-    tag_str = if @output_include_tag
-                tag + @f_separator
-              else
-                ''
-              end
-    time_str + tag_str + stringify_record(record) + "\n"
-  end
-
-end
+require 'fluent/mixin/plaintextformatter'
 
 class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('file_alternative', self)
@@ -215,8 +12,6 @@ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
 
   config_param :path, :string  # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S
 
-  config_param :time_format, :string, :default => nil
-
   config_param :compress, :default => nil do |val|
     c = SUPPORTED_COMPRESS[val.to_sym]
     unless c
@@ -225,14 +20,7 @@ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
     c
   end
 
-  include FluentExt::PlainTextFormatterMixin
-
-  config_set_default :output_include_time, true
-  config_set_default :output_include_tag, true
-  config_set_default :output_data_type, 'json'
-  config_set_default :field_separator, "\t"
-  config_set_default :add_newline, true
-  config_set_default :remove_prefix, nil
+  include Fluent::Mixin::PlainTextFormatter
 
   def initialize
     super
@@ -272,8 +60,6 @@ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
     unless @path.index('/') == 0
       raise Fluent::ConfigError, "Path on filesystem MUST starts with '/', but '#{@path}'"
     end
-
-    @timef = Fluent::TimeFormatter.new(@time_format, @localtime)
   end
 
   def start
@@ -290,10 +76,8 @@ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
     record.to_json
   end
 
-  def format(tag, time, record)
-    time_str = @timef.format(time)
-    time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
-  end
+  # def format(tag, time, record)
+  # end
 
   def path_format(chunk_key)
     suffix = case @compress

data/test/plugin/test_out_file_alternative.rb

@@ -36,7 +36,7 @@ class FileAlternativeOutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.output_include_tag
     assert_equal 'json', d.instance.output_data_type
     assert_equal true, d.instance.add_newline
-    assert_equal "\t", d.instance.field_separator
+    assert_equal "TAB", d.instance.field_separator
     assert_nil d.instance.remove_prefix
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-file-alternative
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-19 00:00:00.000000000 Z
+date: 2012-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-plaintextformatter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -43,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-plaintextformatter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: alternative implementation of out_file, with various configurations
 email:
 - tagomoris@gmail.com