fluent-plugin-forward-aws 0.1.1 → 0.1.2

data/README.md

@@ -42,14 +42,49 @@ In short, change SQS's access policy to accept "SendMessage" from your SNS ARN,
 You can do the above step in one shot from SQS Management Console.  
 For more detail, check [amazon official document](http://docs.aws.amazon.com/sns/latest/gsg/SendMessageToSQS.html)
 
+## Common Configuration
+### Parameters
++ **aws_access_key_id** (required)  
+AWS Acccess Key ID
+
++ **aws_secret_access_key** (required)  
+AWS Secket Access Key
+
++ **aws_s3_endpoint** (required)  
+[s3 Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) for your bucket
+
++ **aws_s3_bucketname** (required)  
+S3 Bucketname
+
++ **aws_s3_skiptest** (optional, default = false)  
+Skip S3 Related test at startup
+
++ **add_tag_prefix** (optional)  
+Add specified prefix to tag before processing log
+
++ **remove_tag_prefix** (optional)  
+Remove specified prefix from tag before processing log
+
++ **channel** (optional, default = "default")  
+Tag that Forward-AWS plugin uses for grouping logs.
+
 ## Out Plugin Configuration
+### Parameters
++ **aws_sns_endpoint** (required)  
+[SNS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sns_region) for your topic
+
++ **aws_sns_topic_arn** (required)  
+SNS Topic ARN
+
++ **aws_sns_skiptest** (optional, default = false)  
+Skip SNS Related test at startup
 
 ###Required AWS permission 
 + s3:PutObject
 + sns:Publish
 
 ### Basic configuration
-Use "default" channel for all the log data
+Use "default" channel for all the log data.  
 ```
 <match **>
   type forward_aws
@@ -62,16 +97,17 @@ Use "default" channel for all the log data
   aws_sns_endpoint      sns.ap-northeast-1.amazonaws.com
   aws_sns_topic_arn     arn:aws:sns:ap-northeast-1:XXXXXXXXXXXXXXXXXXXX
 
-  buffer_type           file
+  # Time Sliced Output options
   buffer_path           /var/log/td-agent/buffer/forward_aws
   time_slice_wait       1m
-  utc
   time_slice_format     %Y/%m/%d/%H/%M
+  utc
+  flush_at_shutdown     true
 </match>
 ```
 
 ### Advanced configuration using forest plugin
-Use tag as channel
+Use tag as forward-AWS channel
 ```
 <match **>
   type forest
@@ -87,16 +123,29 @@ Use tag as channel
     aws_sns_endpoint      sns.ap-northeast-1.amazonaws.com
     aws_sns_topic_arn     arn:aws:sns:ap-northeast-1:XXXXXXXXXXXXXXXXXXXX
 
-    buffer_type           file
+    # Time Sliced Output options
     buffer_path           /var/log/td-agent/buffer/forward_aws-${tag}
     time_slice_wait       1m
-    utc
     time_slice_format     %Y/%m/%d/%H/%M
+    utc
+    flush_at_shutdown     true
   </template>
 </match>
 ```
 
 ## In Plugin Configuration
+### Parameters
++ **aws_sqs_endpoint** (required)  
+[SQS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) for your topic
+
++ **aws_sqs_queue_url** (required)  
+SQS Queue URL (not ARN)
+
++ **aws_sqs_skiptest** (optional, default = false)  
+Skip SQS Related test at startup
+
++ **channelEnableRegEx** (optional, default = false)
+Enabled Regular Expression for channel value.
 
 ###Required AWS permission 
 + s3:GetObject

data/fluent-plugin-forward-aws.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-forward-aws"
-  gem.version       = "0.1.1"
+  gem.version       = "0.1.2"
   gem.authors       = ["Tomohisa Ota"]
   gem.email         = ["tomohisa.ota+github@gmail.com"]
   gem.description   = "Fluentd Forward Plugin using Amazon Web Service"

data/lib/fluent/plugin/forward_aws_util.rb

@@ -0,0 +1,22 @@
+module ForwardAWSUtil
+  
+  def self.filtertag(tag,add_prefix=nil,remove_prefix=nil)
+    # Remove Prefix First
+    if remove_prefix && tag
+      tag = tag.sub(/^#{Regexp.escape(remove_prefix)}\b\.?/,"")
+    end
+    
+    # Add Prefix
+    if add_prefix
+      if tag && tag.length > 0 
+        tag = "#{add_prefix}.#{tag}"
+      else
+        tag = add_prefix
+      end
+    end
+    
+    # Return Result
+    return tag
+  end
+  
+end

data/lib/fluent/plugin/in_forward_aws.rb

@@ -1,8 +1,9 @@
 class Fluent::ForwardAWSInput < Fluent::Input
   Fluent::Plugin.register_input('forward_aws', self)
-
-  # config_param :hoge, :string, :default => 'hoge'
-
+  
+  require_relative "forward_aws_util"
+  include ForwardAWSUtil
+    
   def initialize
     super
     require 'aws-sdk'
@@ -26,6 +27,9 @@ class Fluent::ForwardAWSInput < Fluent::Input
   config_param :aws_sqs_queue_url, :string, :default => nil
   config_param :aws_sqs_skiptest, :bool, :default => false
   
+  config_param :add_tag_prefix, :string, :default => nil
+  config_param :remove_tag_prefix, :string, :default => nil
+  
   # Not documented parameters. Subject to change in future release
   config_param :aws_sqs_process_interval, :integer, :default => 1
   config_param :aws_sqs_monitor_interval, :integer, :default => 10
@@ -135,6 +139,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
           streamUnpacker.feed(reader.read())
           streamUnpacker.each {|event|
             (tag, time, record) = event
+            tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
             Fluent::Engine.emit(tag,time,record)
           }
         }

data/lib/fluent/plugin/out_forward_aws.rb

@@ -1,8 +1,9 @@
 class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('forward_aws', self)
-
-  # config_param :hoge, :string, :default => 'hoge'
-
+  
+  require_relative "forward_aws_util"
+  include ForwardAWSUtil
+    
   config_param :channel, :string, :default => "default"
 
   config_param :aws_access_key_id, :string, :default => nil
@@ -16,6 +17,9 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
   config_param :aws_sns_topic_arn, :string, :default => nil
   config_param :aws_sns_skiptest, :bool, :default => false
 
+  config_param :add_tag_prefix, :string, :default => nil
+  config_param :remove_tag_prefix, :string, :default => nil
+
   # Not documented parameters. Subject to change in future release
   config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
   config_param :aws_sns_emailsubject, :string, :default => "SNS Message"
@@ -86,6 +90,7 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
   end
 
   def format(tag, time, record)
+    tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
     [tag, time, record].to_msgpack
   end
 

data/test/plugin/test_forward_aws_util.rb

@@ -0,0 +1,35 @@
+require 'helper'
+
+class ForwardAWSUtilTest < Test::Unit::TestCase
+  require_relative "../../lib/fluent/plugin/forward_aws_util"
+  include ForwardAWSUtil
+  
+  def test_filtertag
+    assert_equal('aaa',ForwardAWSUtil.filtertag("aaa"))
+    assert_equal('aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb"))
+    assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc"))
+
+    # Add prefix
+    assert_equal('ddd.aaa',ForwardAWSUtil.filtertag("aaa","ddd"))
+    assert_equal('ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ddd"))
+    assert_equal('ddd.aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc","ddd"))
+
+    # Remove prefix
+    assert_equal('',ForwardAWSUtil.filtertag("aaa",nil,"aaa"))
+    assert_equal('bbb',ForwardAWSUtil.filtertag("aaa.bbb",nil,"aaa"))
+    assert_equal('bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"aaa"))
+    
+    # Add and remove
+    assert_equal('ccc.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc","aaa"))
+
+    # Corner cases
+    # Do not remove tag in middle
+    assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"bbb"))
+    # Do not remove incomplete tag
+    assert_equal('aaaa.bbb',ForwardAWSUtil.filtertag("aaaa.bbb",nil,"aaa"))
+    # Remove tag does not affect newly added tag
+    assert_equal('ccc.ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","ccc"))
+    assert_equal('ccc.ddd.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","aaa"))
+    
+  end
+end

data/test/plugin/test_out_foward_aws.rb

@@ -72,6 +72,28 @@ class ForwardAWSOutputTest < Test::Unit::TestCase
     d.expect_format ["test",time,{"a"=>2}].to_msgpack
     d.run
   end
+  
+  def test_format_addprefixtest
+    d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format ["addprefixtest.test",time,{"a"=>1}].to_msgpack
+    d.expect_format ["addprefixtest.test",time,{"a"=>2}].to_msgpack
+    d.run
+  end
+
+  def test_format_removeprefixtest
+    d = create_driver(DUMMYCONFIG + "remove_tag_prefix test")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format ["",time,{"a"=>1}].to_msgpack
+    d.expect_format ["",time,{"a"=>2}].to_msgpack
+    d.run
+  end  
 
   def test_check_aws_s3
     unless(@CONFIG)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-forward-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-27 00:00:00.000000000 Z
+date: 2013-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -56,10 +56,12 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-forward-aws.gemspec
+- lib/fluent/plugin/forward_aws_util.rb
 - lib/fluent/plugin/in_forward_aws.rb
 - lib/fluent/plugin/out_forward_aws.rb
 - test/awsconfig.yml.sample
 - test/helper.rb
+- test/plugin/test_forward_aws_util.rb
 - test/plugin/test_in_foward_aws.rb
 - test/plugin/test_out_foward_aws.rb
 homepage: ''
@@ -89,5 +91,6 @@ summary: Fluentd Forward Plugin using Amazon Web Service
 test_files:
 - test/awsconfig.yml.sample
 - test/helper.rb
+- test/plugin/test_forward_aws_util.rb
 - test/plugin/test_in_foward_aws.rb
 - test/plugin/test_out_foward_aws.rb