fluent-plugin-forward-aws 0.1.3 → 0.1.4

data/README.md

@@ -37,47 +37,32 @@ Create SQS Queue, subscribe to SNS, and memorize following parameters
 + aws_sqs_endpoint
 + aws_sqs_queue_url 
 
-#### How to configure SQS as SNS subscriber
+### How to configure SQS as SNS subscriber
 In short, change SQS's access policy to accept "SendMessage" from your SNS ARN, And add SQS ARN to SNS subscribers. 
 You can do the above step in one shot from SQS Management Console.  
 For more detail, check [amazon official document](http://docs.aws.amazon.com/sns/latest/gsg/SendMessageToSQS.html)
 
 ## Common Configuration
 ### Parameters
-+ **aws_access_key_id** (required)  
-AWS Acccess Key ID
-
-+ **aws_secret_access_key** (required)  
-AWS Secket Access Key
-
-+ **aws_s3_endpoint** (required)  
-[s3 Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) for your bucket
-
-+ **aws_s3_bucketname** (required)  
-S3 Bucketname
-
-+ **aws_s3_skiptest** (optional, default = false)  
-Skip S3 Related test at startup
-
-+ **add_tag_prefix** (optional)  
-Add specified prefix to tag before processing log
-
-+ **remove_tag_prefix** (optional)  
-Remove specified prefix from tag before processing log
-
-+ **channel** (optional, default = "default")  
-Tag that Forward-AWS plugin uses for grouping logs.
+ name                 | type                            | description
+----------------------|---------------------------------|---------------------------
+type                  | string (required)               | type of plugin should be **forward_aws**
+aws_access_key_id     | string (required)               |  AWS Acccess Key ID
+aws_secret_access_key | string (required)               | AWS Secket Access Key
+aws_s3_endpoint       | string (required)               | [s3 Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) for your bucket
+aws_s3_bucketname     | string (required)               | S3 Bucketname
+aws_s3_skiptest       | bool (default = false)          | Skip S3 Related test at startup
+add_tag_prefix        | string (default = nil)          | Add specified prefix to tag before processing log
+remove_tag_prefix     | string (default = nil)          | Remove specified prefix from tag before processing log
+channel               | string (default = "default")    | Tag that Forward-AWS plugin uses for grouping logs.
 
 ## Out Plugin Configuration
 ### Parameters
-+ **aws_sns_endpoint** (required)  
-[SNS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sns_region) for your topic
-
-+ **aws_sns_topic_arn** (required)  
-SNS Topic ARN
-
-+ **aws_sns_skiptest** (optional, default = false)  
-Skip SNS Related test at startup
+ name                 | type                            | description
+----------------------|---------------------------------|---------------------------
+aws_sns_endpoint      | string (required)               | [SNS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sns_region) for your topic
+aws_sns_topic_arn     | string (required)               | SNS Topic ARN
+aws_sns_skiptest      | bool (default = false)          | Skip SNS Related test at startup
 
 ###Required AWS permission 
 + s3:PutObject
@@ -135,17 +120,14 @@ Use tag as forward-AWS channel
 
 ## In Plugin Configuration
 ### Parameters
-+ **aws_sqs_endpoint** (required)  
-[SQS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) for your topic
-
-+ **aws_sqs_queue_url** (required)  
-SQS Queue URL (not ARN)
-
-+ **aws_sqs_skiptest** (optional, default = false)  
-Skip SQS Related test at startup
 
-+ **channelEnableRegEx** (optional, default = false)
-Enabled Regular Expression for channel value.
+ name                 | type                            | description
+----------------------|---------------------------------|---------------------------
+aws_sqs_endpoint      | string (required)               | [SQS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) for your topic
+aws_sqs_queue_url     | string (required)               | SQS Queue URL (not ARN)
+aws_sqs_skiptest      | bool (default = false)          | Skip SQS Related test at startup
+channelEnableRegEx    | bool (default = false)          | Enabled Regular Expression when checking channel
+dry_run               | bool (default = false)          | Do not delete notification after processing
 
 ###Required AWS permission 
 + s3:GetObject

data/fluent-plugin-forward-aws.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-forward-aws"
-  gem.version       = "0.1.3"
+  gem.version       = "0.1.4"
   gem.authors       = ["Tomohisa Ota"]
   gem.email         = ["tomohisa.ota+github@gmail.com"]
   gem.description   = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"

data/lib/fluent/plugin/in_forward_aws.rb

@@ -30,6 +30,8 @@ class Fluent::ForwardAWSInput < Fluent::Input
   config_param :add_tag_prefix, :string, :default => nil
   config_param :remove_tag_prefix, :string, :default => nil
   
+  config_param :dry_run, :bool, :default => false
+  
   # Not documented parameters. Subject to change in future release
   config_param :aws_sqs_process_interval, :integer, :default => 1
   config_param :aws_sqs_monitor_interval, :integer, :default => 10
@@ -70,7 +72,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
     unless(@aws_sqs_skiptest)
       init_aws_sqs_queue()
       begin
-        @queue.receive_message() do |msg|
+        @queue.receive_message() do |notification|
         end
       rescue => e
         raise Fluent::ConfigError.new("Cannot fetch queue from SQS. Need sqs:ReceiveMessage sqs:DeleteMessage permission for resource " + @aws_sqs_queue_url)
@@ -90,10 +92,20 @@ class Fluent::ForwardAWSInput < Fluent::Input
   def run
     @running = true
     while true
-      msg = @queue.receive_message()
-      if msg
-        if(process(JSON.parse(msg.as_sns_message.body)))
-          msg.delete()
+      $log.debug "Polling SQS"
+      notificationRaw = @queue.receive_message()
+      if notificationRaw
+        notification = JSON.parse(notificationRaw.as_sns_message.body)
+        $log.debug "Received Notification#{notification}"
+        if(process(notification))
+          if dry_run
+            $log.info "Notification processed in dry-run mode #{notification}"
+          else
+            notificationRaw.delete()
+            $log.debug "Deleted processed notification #{notification}"
+          end
+        else
+          $log.error "Could not process notification, pending... #{notification}"
         end
         sleep @aws_sqs_process_interval
         @locker.synchronize do
@@ -110,26 +122,33 @@ class Fluent::ForwardAWSInput < Fluent::Input
     puts e 
   end
 
-  def process(msg)
-    if msg["type"] == "ping"
+  def process(notification)
+    if notification["type"] == "ping"
       # Ignore ping message
       return true
     end
-    if msg["type"] == "out"
+    if notification["type"] == "out"
       # Silently ignore non matching logs
-      if msg["bucketname"] != @aws_s3_bucketname
+      if notification["bucketname"] != @aws_s3_bucketname
+        $log.debug "Bucketname does not match. Ignoring"
         return true
       end
       if(@channelEnableRegEx)
-        return true unless Regexp.new(@channel).match(msg["channel"])
+        unless Regexp.new(@channel).match(notification["channel"])
+          $log.debug "Channel RegEx does not match. Ignoring"
+          return true
+        end
       else
-        return true unless @channel == msg["channel"]
+        unless @channel == notification["channel"]
+          $log.debug "Channel does not match. Ignoring"
+          return true
+        end
       end
-      
       tmpFile = Tempfile.new("forward-aws-")
       begin
         #Download log file to temporary file
-        @bucket.objects[msg["path"]].read do |chunk|
+        $log.debug "Download log object from S3 bucket #{@aws_s3_bucketname} path #{notification["path"]}"
+        @bucket.objects[notification["path"]].read do |chunk|
           tmpFile.write(chunk)
         end
         tmpFile.close()

data/lib/fluent/plugin/out_forward_aws.rb

@@ -108,6 +108,7 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
     begin
       chunk.write_to(writer)
       writer.close
+      $log.debug "Upload log object to S3 bucket #{@aws_s3_bucketname} path #{s3path}"
       @bucket.objects[s3path].write(Pathname.new(tmpFile.path), :content_type => 'application/x-gzip')
       notification = {
         "type"        => "out",
@@ -118,6 +119,7 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
         "compression" => compression
       }
       topic = @sns.topics[@aws_sns_topic_arn]
+      $log.debug "Posting notification #{notification}"
       topic.publish(JSON.pretty_generate(notification), :subject => @aws_sns_emailsubject)
     ensure
       writer.close rescue nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-forward-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-28 00:00:00.000000000 Z
+date: 2013-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd