fluent-plugin-forward-aws 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82dcc40302aaebc4f395387518c5bc0ad5b10964
-  data.tar.gz: 0c05f8555f790f65d2472d6bd05065f97d99c060
+  metadata.gz: 6a407c6eeec19a09a1eb1536c8dbb7f21fe5ac7e
+  data.tar.gz: 90449701adb6eb32977b87e2c0c6beba11480ec9
 SHA512:
-  metadata.gz: 6cb2f549924e3a261a33b8d691ae51899d60dcdaa57925f551ead3ee585ded3726465f719451fd6d07d6be38947a16c93ac5dfcc06f0e33907996263b7999953
-  data.tar.gz: afaeaad61e2e74be0f7795382474d1f5ae0097253abd3d3a7f3767eeff778e65d684fae0238bf053d065e86e3dcdf7ea672a127b0e12aa41c152f189c479fc2b
+  metadata.gz: c9874949fc9d9969a16910afc606eb3beb7c6373412eec2138c3fdbd31907054ee36608bc1716b7e554e377c052927ec9a650d00144c4de978ee02d2a934913b
+  data.tar.gz: d91d3db1dc5d8e4c2001f3ea63ebf7985ca2e8789649d90f62e9365642f740b6716b67879920be17de0def6171c2af739e4a416753d8f7f446d8463718f79b56

data/README.md

@@ -121,11 +121,16 @@ Use tag as forward-AWS channel
 ## In Plugin Configuration
 ### Parameters
 
- name                 | type                            | description
-----------------------|---------------------------------|---------------------------
-aws_sqs_endpoint      | string (required)               | [SQS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) for your topic
-aws_sqs_queue_url     | string (required)               | SQS Queue URL (not ARN)
-aws_sqs_skiptest      | bool (default = false)          | Skip SQS Related test at startup
+ name                      | type                            | description
+---------------------------|---------------------------------|---------------------------
+aws_sqs_endpoint           | string (required)               | [SQS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) for your topic
+aws_sqs_queue_url          | string (required)               | SQS Queue URL (not ARN)
+aws_sqs_skiptest           | bool (default = false)          | Skip SQS Related test at startup
+aws_sqs_wait_time_seconds  | integer(default = 5)            | long polling
+aws_sqs_limit              | integer(default = 10)           | The maximum number of messages to receive
+aws_sqs_visibilitiy_timeout| integer(default = 300)          | Duration that the received messages are hidden
+
+
 channelEnableRegEx    | bool (default = false)          | Enabled Regular Expression when checking channel
 dry_run               | bool (default = false)          | Do not delete notification after processing
 
@@ -173,6 +178,10 @@ Use regex to filter channel
 Forward-AWS plugin do not delete buffer objects on S3.  
 Use [S3's lifecycle management](http://docs.aws.amazon.com/AmazonS3/latest/dev/manage-lifecycle-using-console.html) to automatically archive or delete old buffer objects.
 
+### SQS long polling
+forward-AWS uses SQS long polling by default. 
+Long polling could slow down fluentd shutdown sequence.
+
 ### How to use buffer objects as raw input
 Each buffer object is msgpack stream object with gzip compression.
 

data/fluent-plugin-forward-aws.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-forward-aws"
-  gem.version       = "0.1.5"
+  gem.version       = "0.1.6"
   gem.authors       = ["Tomohisa Ota"]
   gem.email         = ["tomohisa.ota+github@gmail.com"]
   gem.description   = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"

data/lib/fluent/plugin/in_forward_aws.rb

@@ -26,6 +26,9 @@ class Fluent::ForwardAWSInput < Fluent::Input
   config_param :aws_sqs_endpoint, :string, :default => nil
   config_param :aws_sqs_queue_url, :string, :default => nil
   config_param :aws_sqs_skiptest, :bool, :default => false
+  config_param :aws_sqs_wait_time_seconds,  :integer, :default => 5
+  config_param :aws_sqs_limit,              :integer, :default => 10
+  config_param :aws_sqs_visibilitiy_timeout,:integer, :default => 300
   
   config_param :add_tag_prefix, :string, :default => nil
   config_param :remove_tag_prefix, :string, :default => nil
@@ -33,9 +36,9 @@ class Fluent::ForwardAWSInput < Fluent::Input
   config_param :dry_run, :bool, :default => false
   
   # Not documented parameters. Subject to change in future release
-  config_param :aws_sqs_process_interval, :integer, :default => 0
-  config_param :aws_sqs_monitor_interval, :integer, :default => 10
-  config_param :aws_sqs_limit,            :integer, :default => 10
+  config_param :aws_sqs_process_interval,   :integer, :default => 0
+  config_param :aws_sqs_monitor_interval,   :integer, :default => 25
+  
   config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
   config_param :start_thread, :bool, :default => true
   
@@ -87,19 +90,25 @@ class Fluent::ForwardAWSInput < Fluent::Input
     init_aws_s3_bucket()
     init_aws_sqs_queue()
     if(@start_thread)
-      @thread = Thread.new(&method(:run))
+      @thread = Thread.new(&method(:run)) unless @thread
     end
   end
 
   def run
     @running = true
     while true
-      $log.debug "Polling SQS"
-      notificationRaws = @queue.receive_message({:limit => @aws_sqs_limit})
+      $log.debug "Long Polling SQS for #{@aws_sqs_wait_time_seconds} secs with message limit #{@aws_sqs_limit}"
+      notificationRaws = @queue.receive_message({
+        :limit               => @aws_sqs_limit,
+        :wait_time_seconds   => @aws_sqs_wait_time_seconds,
+        :visibilitiy_timeout => @aws_sqs_visibilitiy_timeout
+      })
+      $log.debug "Polling finished"
       if(notificationRaws && !notificationRaws.instance_of?(Array))
         notificationRaws = [notificationRaws]
       end
       if notificationRaws && notificationRaws.size != 0
+        $log.debug "Received #{notificationRaws.size} messages"
         notificationRaws.each{ |notificationRaw|
           notification = JSON.parse(notificationRaw.as_sns_message.body)
           $log.debug "Received Notification#{notification}"
@@ -114,12 +123,19 @@ class Fluent::ForwardAWSInput < Fluent::Input
             $log.error "Could not process notification, pending... #{notification}"
           end
         }
+        @locker.synchronize do
+          return unless @running
+        end
         sleep @aws_sqs_process_interval if(@aws_sqs_process_interval > 0)
         @locker.synchronize do
           return unless @running
         end
         next
       end
+      $log.debug "No messages in queue, sleep for #{@aws_sqs_monitor_interval} secs"
+      @locker.synchronize do
+        return unless @running
+      end
       sleep @aws_sqs_monitor_interval
       @locker.synchronize do
         return unless @running
@@ -190,6 +206,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
     super
     # Stop Thread and join
     @locker.synchronize do
+      $log.debug "Stopping thread"
       @running = false
     end
     if(@thread)
@@ -201,17 +218,6 @@ class Fluent::ForwardAWSInput < Fluent::Input
   
   private
   
-  def check_aws_credential
-    unless @aws_access_key_id
-      raise Fluent::ConfigError.new("aws_access_key_id is required")
-    end
-    unless @aws_secret_access_key
-      raise Fluent::ConfigError.new("aws_secret_access_key is required")
-    end
-  end
-  
-  private
-  
   def init_aws_s3_bucket
     unless @bucket
       options = {}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-forward-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Tomohisa Ota
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-04 00:00:00.000000000 Z
+date: 2013-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd