fluent-plugin-forward-aws 0.1.4 → 0.1.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 82dcc40302aaebc4f395387518c5bc0ad5b10964
+  data.tar.gz: 0c05f8555f790f65d2472d6bd05065f97d99c060
+SHA512:
+  metadata.gz: 6cb2f549924e3a261a33b8d691ae51899d60dcdaa57925f551ead3ee585ded3726465f719451fd6d07d6be38947a16c93ac5dfcc06f0e33907996263b7999953
+  data.tar.gz: afaeaad61e2e74be0f7795382474d1f5ae0097253abd3d3a7f3767eeff778e65d684fae0238bf053d065e86e3dcdf7ea672a127b0e12aa41c152f189c479fc2b

data/fluent-plugin-forward-aws.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-forward-aws"
-  gem.version       = "0.1.4"
+  gem.version       = "0.1.5"
   gem.authors       = ["Tomohisa Ota"]
   gem.email         = ["tomohisa.ota+github@gmail.com"]
   gem.description   = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"

data/lib/fluent/plugin/in_forward_aws.rb

@@ -33,11 +33,13 @@ class Fluent::ForwardAWSInput < Fluent::Input
   config_param :dry_run, :bool, :default => false
   
   # Not documented parameters. Subject to change in future release
-  config_param :aws_sqs_process_interval, :integer, :default => 1
+  config_param :aws_sqs_process_interval, :integer, :default => 0
   config_param :aws_sqs_monitor_interval, :integer, :default => 10
+  config_param :aws_sqs_limit,            :integer, :default => 10
   config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
   config_param :start_thread, :bool, :default => true
   
+  
   def configure(conf)
     super
     if /^\s*$/ =~ @channel
@@ -93,21 +95,26 @@ class Fluent::ForwardAWSInput < Fluent::Input
     @running = true
     while true
       $log.debug "Polling SQS"
-      notificationRaw = @queue.receive_message()
-      if notificationRaw
-        notification = JSON.parse(notificationRaw.as_sns_message.body)
-        $log.debug "Received Notification#{notification}"
-        if(process(notification))
-          if dry_run
-            $log.info "Notification processed in dry-run mode #{notification}"
+      notificationRaws = @queue.receive_message({:limit => @aws_sqs_limit})
+      if(notificationRaws && !notificationRaws.instance_of?(Array))
+        notificationRaws = [notificationRaws]
+      end
+      if notificationRaws && notificationRaws.size != 0
+        notificationRaws.each{ |notificationRaw|
+          notification = JSON.parse(notificationRaw.as_sns_message.body)
+          $log.debug "Received Notification#{notification}"
+          if(process(notification))
+            if dry_run
+              $log.info "Notification processed in dry-run mode #{notification}"
+            else
+              notificationRaw.delete()
+              $log.debug "Deleted processed notification #{notification}"
+            end
           else
-            notificationRaw.delete()
-            $log.debug "Deleted processed notification #{notification}"
+            $log.error "Could not process notification, pending... #{notification}"
           end
-        else
-          $log.error "Could not process notification, pending... #{notification}"
-        end
-        sleep @aws_sqs_process_interval
+        }
+        sleep @aws_sqs_process_interval if(@aws_sqs_process_interval > 0)
         @locker.synchronize do
           return unless @running
         end
@@ -163,6 +170,14 @@ class Fluent::ForwardAWSInput < Fluent::Input
           }
         }
         return true
+      rescue => e
+        if(e.message == "Access Denied")
+          $log.warn "Access Denied for key #{notification["path"]}"
+          # Object may have been deleted. Do not retry
+          return true
+        else
+          $log.error e
+        end
       ensure
         tmp.close(true) rescue nil
       end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-forward-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.4
-  prerelease: 
+  version: 0.1.5
 platform: ruby
 authors:
 - Tomohisa Ota
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-01 00:00:00.000000000 Z
+date: 2013-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -66,27 +61,26 @@ files:
 - test/plugin/test_out_foward_aws.rb
 homepage: http://github.com/tomohisaota/fluent-plugin-forward-aws
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)
 test_files:
 - test/awsconfig.yml.sample