circuitry 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c5939511fb1a459b3a6674d0adccd6138235674
-  data.tar.gz: 25c640746707e9b6a34fc016110a8b5069ad730b
+  metadata.gz: aacd53812987507821869c3aee366956a7f824ba
+  data.tar.gz: a5c069dc6b82c398b89d06dd3aa826fc8191869e
 SHA512:
-  metadata.gz: 2119226fb89515758f04f426bf5b1d8e5576e27a7a1e12108ad35b8b0e03dc0085d42184095a2dd8a347d7c5a58cd3b6409edb55fa5d661bc29f8958ba8125e5
-  data.tar.gz: f1a5eadbfcf85445908d6c333eed171ac49e3e28d851a949460dd84f5282a67150ce244c8f3ea13c23f7ff708e479e2ee5966857cba827a4353b4fa8261c030f
+  metadata.gz: be97c0b2b282f712eefe048a4da9c412f7e85df3921e89ca5aa9829bdf1a3f4700dfeb60812e131001790da5e2948ecd9c66e419f2c09b6a04f6969b145985c1
+  data.tar.gz: 0ae5a71aac398cdba9bfa9a3a8771cc0eb6e6455088f7fb9459e1941f41916b607456aa7d013e09ffe5f09e3f68290f7983b96005d9f26234acc44379a4ccc73

data/CHANGELOG.md

@@ -0,0 +1,40 @@
+## Circuitry 1.4.0 (Nov 15, 2015)
+
+* Replace [fog-aws](https://github.com/fog/fog-aws) with
+  [aws-sdk](https://github.com/aws/aws-sdk-ruby). *Matt Huggins*
+* Fix long polling for subscriber. *Matt Huggins*
+* Retry message deletion if it fails after successful message processing. *Matt Huggins*
+
+## Circuitry 1.3.1 (Nov 6, 2015)
+
+* Implement redis connection pooling for redis lock strategy. *Matt Huggins*
+
+## Circuitry 1.3.0 (Oct 9, 2015)
+
+* Implement thread exit and fork exit hooks. *Matt Huggins*
+
+## Circuitry 1.2.3 (Aug 26, 2015)
+
+* Treat connection resets as temporary service errors. *Matt Huggins*
+
+## Circuitry 1.2.2 (Aug 13, 2015)
+
+* Ignore temporary service errors. *Matt Huggins*
+
+## Circuitry 1.2.1 (Jul 29, 2015)
+
+* Unlock soft locks from messages that were unsuccessfully processed. *Matt Huggins*
+
+## Circuitry 1.2.0 (Jul 20, 2015)
+
+* Implement lock strategies to prevent duplicate message processing. *Matt Huggins*
+* Implement timeout for message processing on both subscriber and publisher. *Matt Huggins*
+
+## Circuitry 1.1.0 (Jul 8, 2015)
+
+* Permit forking, threading, and batching async strategies for both subscriber and publisher.
+  *Matt Huggins*
+
+## Circuitry 1.0.0 (Jun 25, 2015)
+
+* Initial release. *Matt Huggins*

data/README.md

@@ -410,6 +410,7 @@ and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 1. Fork it ( https://github.com/kapost/circuitry/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+3. Update the changelog
+4. Commit your changes (`git commit -am 'Add some feature'`)
+5. Push to the branch (`git push origin my-new-feature`)
+6. Create a new Pull Request

data/circuitry.gemspec

@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'fog-aws', '~> 0.4'
+  spec.add_dependency 'aws-sdk', '~> 2'
+  spec.add_dependency 'retries', '~> 0.0.5'
   spec.add_dependency 'virtus', '~> 1.0'
 
   spec.add_development_dependency 'pry-byebug'
@@ -32,4 +33,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'mock_redis'
   spec.add_development_dependency 'dalli'
   spec.add_development_dependency 'memcache_mock'
+  spec.add_development_dependency 'connection_pool'
 end

data/lib/circuitry/configuration.rb

@@ -28,9 +28,9 @@ module Circuitry
 
     def aws_options
       {
-          aws_access_key_id:     access_key,
-          aws_secret_access_key: secret_key,
-          region:                region,
+          access_key_id:     access_key,
+          secret_access_key: secret_key,
+          region:            region,
       }
     end
 

data/lib/circuitry/message.rb

@@ -3,14 +3,14 @@ require 'circuitry/topic'
 
 module Circuitry
   class Message
-    attr_reader :raw
+    attr_reader :sqs_message
 
-    def initialize(raw)
-      @raw = raw
+    def initialize(sqs_message)
+      @sqs_message = sqs_message
     end
 
     def context
-      @context ||= JSON.parse(raw['Body'])
+      @context ||= JSON.parse(sqs_message.body)
     end
 
     def body
@@ -22,11 +22,11 @@ module Circuitry
     end
 
     def id
-      raw['MessageId']
+      sqs_message.message_id
     end
 
     def receipt_handle
-      raw['ReceiptHandle']
+      sqs_message.receipt_handle
     end
   end
 end

data/lib/circuitry/publisher.rb

@@ -28,16 +28,12 @@ module Circuitry
     def publish(topic_name, object)
       raise ArgumentError.new('topic_name cannot be nil') if topic_name.nil?
       raise ArgumentError.new('object cannot be nil') if object.nil?
-
-      unless can_publish?
-        logger.warn('Circuitry unable to publish: AWS configuration is not set.')
-        return
-      end
+      raise PublishError.new('AWS configuration is not set') unless can_publish?
 
       process = -> do
         Timeout.timeout(timeout) do
           topic = TopicCreator.find_or_create(topic_name)
-          sns.publish(topic.arn, object.to_json)
+          sns.publish(topic_arn: topic.arn, message: object.to_json)
         end
       end
 

data/lib/circuitry/services/sns.rb

@@ -1,10 +1,10 @@
-require 'fog/aws'
+require 'aws-sdk'
 
 module Circuitry
   module Services
     module SNS
       def sns
-        @sns ||= Fog::AWS::SNS.new(Circuitry.config.aws_options)
+        @sns ||= Aws::SNS::Client.new(Circuitry.config.aws_options)
       end
     end
   end

data/lib/circuitry/services/sqs.rb

@@ -1,10 +1,10 @@
-require 'fog/aws'
+require 'aws-sdk'
 
 module Circuitry
   module Services
     module SQS
       def sqs
-        @sqs ||= Fog::AWS::SQS.new(Circuitry.config.aws_options)
+        @sqs ||= Aws::SQS::Client.new(Circuitry.config.aws_options)
       end
     end
   end

data/lib/circuitry/subscriber.rb

@@ -1,3 +1,4 @@
+require 'retries'
 require 'timeout'
 require 'circuitry/concerns/async'
 require 'circuitry/services/sqs'
@@ -21,14 +22,7 @@ module Circuitry
     }.freeze
 
     CONNECTION_ERRORS = [
-        Excon::Errors::Forbidden,
-    ].freeze
-
-    TEMPORARY_ERRORS = [
-        Excon::Errors::InternalServerError,
-        Excon::Errors::ServiceUnavailable,
-        Excon::Errors::SocketError,
-        Excon::Errors::Timeout,
+        Aws::SQS::Errors::ServiceError,
     ].freeze
 
     def initialize(queue, options = {})
@@ -36,30 +30,35 @@ module Circuitry
 
       options = DEFAULT_OPTIONS.merge(options)
 
+      self.subscribed = false
       self.queue = queue
       self.lock = options[:lock]
       self.async = options[:async]
       self.timeout = options[:timeout]
       self.wait_time = options[:wait_time]
       self.batch_size = options[:batch_size]
+
+      trap_signals
     end
 
     def subscribe(&block)
       raise ArgumentError.new('block required') if block.nil?
+      raise SubscribeError.new('AWS configuration is not set') unless can_subscribe?
 
-      unless can_subscribe?
-        logger.warn('Circuitry unable to subscribe: AWS configuration is not set.')
-        return
-      end
+      logger.info("Subscribing to queue: #{queue}")
 
-      loop do
-        begin
-          receive_messages(&block)
-        rescue *CONNECTION_ERRORS => e
-          logger.error("Connection error to #{queue}: #{e}")
-          raise SubscribeError.new(e)
-        end
-      end
+      self.subscribed = true
+      poll(&block)
+      self.subscribed = false
+
+      logger.info("Unsubscribed from queue: #{queue}")
+    rescue *CONNECTION_ERRORS => e
+      logger.error("Connection error to queue: #{queue}: #{e}")
+      raise SubscribeError.new(e)
+    end
+
+    def subscribed?
+      subscribed
     end
 
     def self.async_strategies
@@ -73,6 +72,7 @@ module Circuitry
     protected
 
     attr_writer :queue, :timeout, :wait_time, :batch_size
+    attr_accessor :subscribed
 
     def lock=(value)
       value = case value
@@ -87,19 +87,28 @@ module Circuitry
 
     private
 
-    def receive_messages(&block)
-      response = nil
+    def trap_signals
+      trap('SIGINT') do
+        if subscribed?
+          Thread.new { logger.info('Interrupt received, unsubscribing from queue...') }
+          self.subscribed = false
+        end
+      end
+    end
+
+    def poll(&block)
+      poller = Aws::SQS::QueuePoller.new(queue, client: sqs)
 
-      begin
-        response = sqs.receive_message(queue, 'MaxNumberOfMessages' => batch_size, 'WaitTimeSeconds' => wait_time)
-      rescue *TEMPORARY_ERRORS => e
-        logger.info("Temporary issue connecting to SQS: #{e.message}")
-        return
+      poller.before_request do |_stats|
+        throw :stop_polling unless subscribed?
       end
 
-      messages = response.body['Message']
-      return if messages.empty?
+      poller.poll(max_number_of_messages: batch_size, wait_time_seconds: wait_time, skip_delete: true) do |messages|
+        process_messages(Array(messages), &block)
+      end
+    end
 
+    def process_messages(messages, &block)
       messages.each do |message|
         process = -> do
           process_message(message, &block)
@@ -144,7 +153,7 @@ module Circuitry
 
     def delete_message(message)
       logger.info("Removing message #{message.id} from queue")
-      sqs.delete_message(queue, message.receipt_handle)
+      sqs.delete_message(queue_url: queue, receipt_handle: message.receipt_handle)
     end
 
     def logger

data/lib/circuitry/topic_creator.rb

@@ -2,8 +2,6 @@ require 'circuitry/services/sns'
 require 'circuitry/topic'
 
 module Circuitry
-  class TopicCreatorError < StandardError; end
-
   class TopicCreator
     include Services::SNS
 
@@ -20,9 +18,8 @@ module Circuitry
     def topic
       return @topic if defined?(@topic)
 
-      response = sns.create_topic(topic_name)
-      arn = response.body.fetch('TopicArn') { raise TopicCreatorError.new('No TopicArn returned from SNS') }
-      @topic = Topic.new(arn)
+      response = sns.create_topic(name: topic_name)
+      @topic = Topic.new(response.topic_arn)
     end
   end
 end

data/lib/circuitry/version.rb

@@ -1,3 +1,3 @@
 module Circuitry
-  VERSION = '1.3.1'
+  VERSION = '1.4.0'
 end

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: circuitry
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Matt Huggins
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-06 00:00:00.000000000 Z
+date: 2015-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fog-aws
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: retries
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
 - !ruby/object:Gem::Dependency
   name: virtus
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon SNS publishing and SQS queue processing.
 email:
 - matt.huggins@kapost.com
@@ -185,9 +213,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".codeclimate.yml"
 - ".gitignore"
 - ".rspec"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md

data/.codeclimate.yml

@@ -1,9 +0,0 @@
----
-engines:
-  rubocop:
-    enabled: true
-ratings:
-  paths:
-  - "**.rb"
-exclude_paths:
-- spec/**/*