logstash-input-jsqs 0.9.1-java → 0.9.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85191dce09f154911ff8d1a2b5d076472bf275c6
-  data.tar.gz: 6bac1f35c24c0f04b87072a1e3d359ab742f1e45
+  metadata.gz: c7035e8063acc3862a7b7289f94cff69691ec567
+  data.tar.gz: b9fe883cf27a4d52f0e829c41dfbba59d96c0508
 SHA512:
-  metadata.gz: 86639ab9e5ee0bd507d407ac76d5d0e98eb8ceb299ddb39a0ab442f41d2423173cbca96e44b132855537fac0fdb57abc953472b64513d7a6a1e7af544042cf79
-  data.tar.gz: 52b578a50de497436eaaa8a708a9d8230942290c7fef80414bbc82a832660795b77a64231da2469d9efac9857c31d4cf8f9a056e7c96390d34dbe981b37464dc
+  metadata.gz: c1d08cca6ac2023c98d47c5af70b698ba5aa3c9992b8cfd1cbd12661b9b38ccf0ed0905ad552eeccff32f31c498f686f2c89cf24e6f5edb275d2084aef219ea2
+  data.tar.gz: e66f13d7a80cd27f50bbf8796e8f705b5ad702e25ee41f0116fdd9d2db1f8ed47a708372f4c8e5a1b3d1df886a7d20c954700902a93e569e3b441c1c0c6f1a4c

data/lib/logstash/inputs/jsqs.rb

@@ -1,20 +1,9 @@
 # encoding: utf-8
+# Original plugin by Al Belsky (https://logstash.jira.com/browse/LOGSTASH-1968)
+
 require "logstash/inputs/threadable"
-require "logstash/namespace"
-require "logstash/plugin_mixins/aws_config"
-require "digest/sha2"
 require 'logstash-input-jsqs_jars.rb'
 
-# aws-java-sdk dependencies
-# require 'jar/aws-java-sdk.jar'
-# require 'jar/httpclient-4.2.jar'
-# require 'jar/httpcore-4.2.jar'
-# require 'jar/commons-codec-1.6'
-# require 'jar/jackson-core-2.1.1.jar'
-# require 'jar/jackson-databind-2.1.1.jar'
-# require 'jar/jackson-annotations-2.1.1.jar'
-#
-
 # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
 #
 # SQS is a simple, scalable queue system that is part of the 
@@ -68,14 +57,6 @@ require 'logstash-input-jsqs_jars.rb'
 # See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
 #
 
-# -------------------------------------
-# NOTICE:
-# This is a custom SQS input that uses the Java AWS SDK (Ruby SDK didn't scale beyond 200 messages per second).
-# This plugin assumes that the logstash distribution contains the Java AWS SDK its required dependencies.
-# In Logstash 1.3.3, Apache HTTP client (4.1) had to be removed from the Logstash distribution,
-#   as it was incompatible with the Apache HTTP client 4.2 version used by the Java AWS SDK.
-# -------------------------------------
-
 class LogStash::Inputs::JSQS < LogStash::Inputs::Threadable
 
   config_name "jsqs"
@@ -89,35 +70,30 @@ class LogStash::Inputs::JSQS < LogStash::Inputs::Threadable
   config :max_inflight_receive_batches, :validate => :number, :default => 50
   config :max_done_receive_batches, :validate => :number, :default => 50
   config :max_number_of_messages, :validate => :number, :default => 10
+  config :retry_count, :validate => :number, :default => 5
 
   @receiveRequest
 
   public
   def register
-
     @logger.info("Registering SQS input", :queue => @queue)
 
-    begin
-        # Client config
-        @logger.debug("Creating AWS SQS queue client", :queue => @queue)
-        clientConfig = ClientConfiguration.new.withMaxConnections(@max_connections)
-        # SQS client
-        @sqs = AmazonSQSAsyncClient.new(clientConfig)
-        @logger.debug("Amazon SQS Client created")
-    
-        # Buffered client config
-        queueBufferConfig = QueueBufferConfig.new.withMaxBatchOpenMs(@max_batch_open_ms).withMaxInflightReceiveBatches(@max_inflight_receive_batches).withMaxDoneReceiveBatches(@max_done_receive_batches)
-
-        @bufferedSqs = AmazonSQSBufferedAsyncClient.new(@sqs, queueBufferConfig);
-        @logger.info("Connected to AWS SQS queue successfully.", :queue => @queue)
-        
-        @receiveRequest = ReceiveMessageRequest.new(@queueUrl).withMaxNumberOfMessages(@max_number_of_messages)
-        
-    rescue Exception => e
-      @logger.error("Unable to access SQS queue.", :error => e.to_s, :queue => @queue)
-      throw e
-    end # begin/rescue
-    
+    # Client config
+    @logger.debug("Creating AWS SQS queue client", :queue => @queue)
+    clientConfig = ClientConfiguration.new.withMaxConnections(@max_connections)
+
+    # SQS client
+    @sqs = AmazonSQSAsyncClient.new(clientConfig)
+    @logger.debug("Amazon SQS Client created")
+
+    # Buffered client config
+    queueBufferConfig = QueueBufferConfig.new.withMaxBatchOpenMs(@max_batch_open_ms).withMaxInflightReceiveBatches(@max_inflight_receive_batches).withMaxDoneReceiveBatches(@max_done_receive_batches)
+
+    @bufferedSqs = AmazonSQSBufferedAsyncClient.new(@sqs, queueBufferConfig);
+    @logger.info("Connected to AWS SQS queue successfully.", :queue => @queue)
+
+    @receiveRequest = ReceiveMessageRequest.new(@queueUrl).withMaxNumberOfMessages(@max_number_of_messages)
+
   end # def register
 
   public
@@ -125,6 +101,7 @@ class LogStash::Inputs::JSQS < LogStash::Inputs::Threadable
     @logger.debug("Polling SQS queue", :queue => @queue)
 
     while running?   
+      begin
         result = @bufferedSqs.receiveMessage(@receiveRequest)            
         deleteEntries = [] 
         # Process messages (expected 0 - 10 messages)
@@ -143,7 +120,16 @@ class LogStash::Inputs::JSQS < LogStash::Inputs::Threadable
             # Issue delete request
             @bufferedSqs.deleteMessageBatch(deleteRequest);            
         end # end if
-        
+      rescue Exception => e
+        if (@retry_count -= 1) > 0
+          @logger.warn("Unable to access SQS queue. Sleeping before retrying.", :error => e.to_s, :queue => @queue)
+          sleep(10)
+          retry
+        else
+          @logger.error("Unable to access SQS queue. Aborting.", :error => e.to_s, :queue => @queue)
+         teardown
+        end # if
+      end # begin
     end # polling loop
   end # def run
 

data/logstash-input-jsqs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.platform = RUBY_PLATFORM
   s.name = 'logstash-input-jsqs'
-  s.version = '0.9.1'
+  s.version = '0.9.2'
   s.licenses = ['Apache License (2.0)']
   s.summary = "SQS input plugin using the AWS Java SDK"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-jsqs. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jsqs
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: java
 authors:
 - Jamie Cressey
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-04 00:00:00.000000000 Z
+date: 2015-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,14 +62,9 @@ files:
 - logstash-input-jsqs.gemspec
 - spec/inputs/example_spec.rb
 - vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.7.13.jar
-- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
 - vendor/jar-dependencies/runtime-jars/commons-logging-1.2.jar
-- vendor/jar-dependencies/runtime-jars/httpclient-4.4.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.4.1.jar
 - vendor/jar-dependencies/runtime-jars/httpcore-4.4.1.jar
-- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.1.jar
-- vendor/jar-dependencies/runtime-jars/jackson-core-2.5.1.jar
-- vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.1.jar
-- vendor/jar-dependencies/runtime-jars/joda-time-2.7.jar
 homepage:
 licenses:
 - Apache License (2.0)