fluent-plugin-forward-aws 0.1.6 → 0.1.7

data/README.md

@@ -52,10 +52,17 @@ aws_secret_access_key | string (required)               | AWS Secket Access Key
 aws_s3_endpoint       | string (required)               | [s3 Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) for your bucket
 aws_s3_bucketname     | string (required)               | S3 Bucketname
 aws_s3_skiptest       | bool (default = false)          | Skip S3 Related test at startup
-add_tag_prefix        | string (default = nil)          | Add specified prefix to tag before processing log
-remove_tag_prefix     | string (default = nil)          | Remove specified prefix from tag before processing log
 channel               | string (default = "default")    | Tag that Forward-AWS plugin uses for grouping logs.
 
+### Parameters by supported by HandleTagNameMixin
+ name                 | type                            
+----------------------|---------------------
+remove_tag_prefix     | string (default = nil)
+add_tag_prefix        | string (default = nil)
+remove_tag_suffix     | string (default = nil)
+add_tag_suffix        | string (default = nil)
+
+
 ## Out Plugin Configuration
 ### Parameters
  name                 | type                            | description

data/fluent-plugin-forward-aws.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-forward-aws"
-  gem.version       = "0.1.6"
+  gem.version       = "0.1.7"
   gem.authors       = ["Tomohisa Ota"]
   gem.email         = ["tomohisa.ota+github@gmail.com"]
   gem.description   = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"

data/lib/fluent/plugin/in_forward_aws.rb

@@ -1,9 +1,20 @@
 class Fluent::ForwardAWSInput < Fluent::Input
   Fluent::Plugin.register_input('forward_aws', self)
   
-  require_relative "forward_aws_util"
-  include ForwardAWSUtil
-    
+  include Fluent::HandleTagNameMixin
+  
+  # Workaround for HandleTagNameMixin bug
+  # https://github.com/fluent/fluentd/pull/109
+  def format_stream(tag, es)
+    out = ''
+    es.each {|time,record|
+      tag_temp = String.new(tag)
+      filter_record(tag_temp, time, record)
+      out << format(tag_temp, time, record)
+    }
+    out
+  end
+     
   def initialize
     super
     require 'aws-sdk'
@@ -29,10 +40,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
   config_param :aws_sqs_wait_time_seconds,  :integer, :default => 5
   config_param :aws_sqs_limit,              :integer, :default => 10
   config_param :aws_sqs_visibilitiy_timeout,:integer, :default => 300
-  
-  config_param :add_tag_prefix, :string, :default => nil
-  config_param :remove_tag_prefix, :string, :default => nil
-  
+    
   config_param :dry_run, :bool, :default => false
   
   # Not documented parameters. Subject to change in future release
@@ -181,7 +189,8 @@ class Fluent::ForwardAWSInput < Fluent::Input
           streamUnpacker.feed(reader.read())
           streamUnpacker.each {|event|
             (tag, time, record) = event
-            tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
+            # Apply HandleTagNameMixin manually
+            filter_record(tag, time, record)
             Fluent::Engine.emit(tag,time,record)
           }
         }

data/lib/fluent/plugin/out_forward_aws.rb

@@ -1,9 +1,20 @@
 class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('forward_aws', self)
   
-  require_relative "forward_aws_util"
-  include ForwardAWSUtil
+  include Fluent::HandleTagNameMixin
     
+  # Workaround for HandleTagNameMixin bug
+  # https://github.com/fluent/fluentd/pull/109
+  def format_stream(tag, es)
+    out = ''
+    es.each {|time,record|
+      tag_temp = String.new(tag)
+      filter_record(tag_temp, time, record)
+      out << format(tag_temp, time, record)
+    }
+    out
+  end
+  
   config_param :channel, :string, :default => "default"
 
   config_param :aws_access_key_id, :string, :default => nil
@@ -17,9 +28,6 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
   config_param :aws_sns_topic_arn, :string, :default => nil
   config_param :aws_sns_skiptest, :bool, :default => false
 
-  config_param :add_tag_prefix, :string, :default => nil
-  config_param :remove_tag_prefix, :string, :default => nil
-
   # Not documented parameters. Subject to change in future release
   config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
   config_param :aws_sns_emailsubject, :string, :default => "SNS Message"
@@ -90,7 +98,6 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
   end
 
   def format(tag, time, record)
-    tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
     [tag, time, record].to_msgpack
   end
 
@@ -126,7 +133,7 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
       tmp.close(true) rescue nil
     end
   end
-  
+    
   private
   
   def init_aws_s3_bucket

data/test/plugin/test_out_foward_aws.rb

@@ -74,7 +74,7 @@ class ForwardAWSOutputTest < Test::Unit::TestCase
   end
   
   def test_format_addprefixtest
-    d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest")
+    d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest.")
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     d.emit({"a"=>1}, time)
     d.emit({"a"=>2}, time)

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-forward-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
+  prerelease: 
 platform: ruby
 authors:
 - Tomohisa Ota
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-05 00:00:00.000000000 Z
+date: 2013-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -51,40 +56,38 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-forward-aws.gemspec
-- lib/fluent/plugin/forward_aws_util.rb
 - lib/fluent/plugin/in_forward_aws.rb
 - lib/fluent/plugin/out_forward_aws.rb
 - test/awsconfig.yml.sample
 - test/helper.rb
-- test/plugin/test_forward_aws_util.rb
 - test/plugin/test_in_foward_aws.rb
 - test/plugin/test_out_foward_aws.rb
 homepage: http://github.com/tomohisaota/fluent-plugin-forward-aws
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)
 test_files:
 - test/awsconfig.yml.sample
 - test/helper.rb
-- test/plugin/test_forward_aws_util.rb
 - test/plugin/test_in_foward_aws.rb
 - test/plugin/test_out_foward_aws.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 6a407c6eeec19a09a1eb1536c8dbb7f21fe5ac7e
-  data.tar.gz: 90449701adb6eb32977b87e2c0c6beba11480ec9
-SHA512:
-  metadata.gz: c9874949fc9d9969a16910afc606eb3beb7c6373412eec2138c3fdbd31907054ee36608bc1716b7e554e377c052927ec9a650d00144c4de978ee02d2a934913b
-  data.tar.gz: d91d3db1dc5d8e4c2001f3ea63ebf7985ca2e8789649d90f62e9365642f740b6716b67879920be17de0def6171c2af739e4a416753d8f7f446d8463718f79b56

data/lib/fluent/plugin/forward_aws_util.rb

@@ -1,22 +0,0 @@
-module ForwardAWSUtil
-  
-  def self.filtertag(tag,add_prefix=nil,remove_prefix=nil)
-    # Remove Prefix First
-    if remove_prefix && tag
-      tag = tag.sub(/^#{Regexp.escape(remove_prefix)}\b\.?/,"")
-    end
-    
-    # Add Prefix
-    if add_prefix
-      if tag && tag.length > 0 
-        tag = "#{add_prefix}.#{tag}"
-      else
-        tag = add_prefix
-      end
-    end
-    
-    # Return Result
-    return tag
-  end
-  
-end

data/test/plugin/test_forward_aws_util.rb

@@ -1,35 +0,0 @@
-require 'helper'
-
-class ForwardAWSUtilTest < Test::Unit::TestCase
-  require_relative "../../lib/fluent/plugin/forward_aws_util"
-  include ForwardAWSUtil
-  
-  def test_filtertag
-    assert_equal('aaa',ForwardAWSUtil.filtertag("aaa"))
-    assert_equal('aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb"))
-    assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc"))
-
-    # Add prefix
-    assert_equal('ddd.aaa',ForwardAWSUtil.filtertag("aaa","ddd"))
-    assert_equal('ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ddd"))
-    assert_equal('ddd.aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc","ddd"))
-
-    # Remove prefix
-    assert_equal('',ForwardAWSUtil.filtertag("aaa",nil,"aaa"))
-    assert_equal('bbb',ForwardAWSUtil.filtertag("aaa.bbb",nil,"aaa"))
-    assert_equal('bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"aaa"))
-    
-    # Add and remove
-    assert_equal('ccc.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc","aaa"))
-
-    # Corner cases
-    # Do not remove tag in middle
-    assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"bbb"))
-    # Do not remove incomplete tag
-    assert_equal('aaaa.bbb',ForwardAWSUtil.filtertag("aaaa.bbb",nil,"aaa"))
-    # Remove tag does not affect newly added tag
-    assert_equal('ccc.ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","ccc"))
-    assert_equal('ccc.ddd.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","aaa"))
-    
-  end
-end