circuitry 1.3.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2c5939511fb1a459b3a6674d0adccd6138235674
4
- data.tar.gz: 25c640746707e9b6a34fc016110a8b5069ad730b
3
+ metadata.gz: aacd53812987507821869c3aee366956a7f824ba
4
+ data.tar.gz: a5c069dc6b82c398b89d06dd3aa826fc8191869e
5
5
  SHA512:
6
- metadata.gz: 2119226fb89515758f04f426bf5b1d8e5576e27a7a1e12108ad35b8b0e03dc0085d42184095a2dd8a347d7c5a58cd3b6409edb55fa5d661bc29f8958ba8125e5
7
- data.tar.gz: f1a5eadbfcf85445908d6c333eed171ac49e3e28d851a949460dd84f5282a67150ce244c8f3ea13c23f7ff708e479e2ee5966857cba827a4353b4fa8261c030f
6
+ metadata.gz: be97c0b2b282f712eefe048a4da9c412f7e85df3921e89ca5aa9829bdf1a3f4700dfeb60812e131001790da5e2948ecd9c66e419f2c09b6a04f6969b145985c1
7
+ data.tar.gz: 0ae5a71aac398cdba9bfa9a3a8771cc0eb6e6455088f7fb9459e1941f41916b607456aa7d013e09ffe5f09e3f68290f7983b96005d9f26234acc44379a4ccc73
@@ -0,0 +1,40 @@
1
+ ## Circuitry 1.4.0 (Nov 15, 2015)
2
+
3
+ * Replace [fog-aws](https://github.com/fog/fog-aws) with
4
+ [aws-sdk](https://github.com/aws/aws-sdk-ruby). *Matt Huggins*
5
+ * Fix long polling for subscriber. *Matt Huggins*
6
+ * Retry message deletion if it fails after successful message processing. *Matt Huggins*
7
+
8
+ ## Circuitry 1.3.1 (Nov 6, 2015)
9
+
10
+ * Implement redis connection pooling for redis lock strategy. *Matt Huggins*
11
+
12
+ ## Circuitry 1.3.0 (Oct 9, 2015)
13
+
14
+ * Implement thread exit and fork exit hooks. *Matt Huggins*
15
+
16
+ ## Circuitry 1.2.3 (Aug 26, 2015)
17
+
18
+ * Treat connection resets as temporary service errors. *Matt Huggins*
19
+
20
+ ## Circuitry 1.2.2 (Aug 13, 2015)
21
+
22
+ * Ignore temporary service errors. *Matt Huggins*
23
+
24
+ ## Circuitry 1.2.1 (Jul 29, 2015)
25
+
26
+ * Unlock soft locks from messages that were unsuccessfully processed. *Matt Huggins*
27
+
28
+ ## Circuitry 1.2.0 (Jul 20, 2015)
29
+
30
+ * Implement lock strategies to prevent duplicate message processing. *Matt Huggins*
31
+ * Implement timeout for message processing on both subscriber and publisher. *Matt Huggins*
32
+
33
+ ## Circuitry 1.1.0 (Jul 8, 2015)
34
+
35
+ * Permit forking, threading, and batching async strategies for both subscriber and publisher.
36
+ *Matt Huggins*
37
+
38
+ ## Circuitry 1.0.0 (Jun 25, 2015)
39
+
40
+ * Initial release. *Matt Huggins*
data/README.md CHANGED
@@ -410,6 +410,7 @@ and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
410
410
 
411
411
  1. Fork it ( https://github.com/kapost/circuitry/fork )
412
412
  2. Create your feature branch (`git checkout -b my-new-feature`)
413
- 3. Commit your changes (`git commit -am 'Add some feature'`)
414
- 4. Push to the branch (`git push origin my-new-feature`)
415
- 5. Create a new Pull Request
413
+ 3. Update the changelog
414
+ 4. Commit your changes (`git commit -am 'Add some feature'`)
415
+ 5. Push to the branch (`git push origin my-new-feature`)
416
+ 6. Create a new Pull Request
@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
19
19
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20
20
  spec.require_paths = ['lib']
21
21
 
22
- spec.add_dependency 'fog-aws', '~> 0.4'
22
+ spec.add_dependency 'aws-sdk', '~> 2'
23
+ spec.add_dependency 'retries', '~> 0.0.5'
23
24
  spec.add_dependency 'virtus', '~> 1.0'
24
25
 
25
26
  spec.add_development_dependency 'pry-byebug'
@@ -32,4 +33,5 @@ Gem::Specification.new do |spec|
32
33
  spec.add_development_dependency 'mock_redis'
33
34
  spec.add_development_dependency 'dalli'
34
35
  spec.add_development_dependency 'memcache_mock'
36
+ spec.add_development_dependency 'connection_pool'
35
37
  end
@@ -28,9 +28,9 @@ module Circuitry
28
28
 
29
29
  def aws_options
30
30
  {
31
- aws_access_key_id: access_key,
32
- aws_secret_access_key: secret_key,
33
- region: region,
31
+ access_key_id: access_key,
32
+ secret_access_key: secret_key,
33
+ region: region,
34
34
  }
35
35
  end
36
36
 
@@ -3,14 +3,14 @@ require 'circuitry/topic'
3
3
 
4
4
  module Circuitry
5
5
  class Message
6
- attr_reader :raw
6
+ attr_reader :sqs_message
7
7
 
8
- def initialize(raw)
9
- @raw = raw
8
+ def initialize(sqs_message)
9
+ @sqs_message = sqs_message
10
10
  end
11
11
 
12
12
  def context
13
- @context ||= JSON.parse(raw['Body'])
13
+ @context ||= JSON.parse(sqs_message.body)
14
14
  end
15
15
 
16
16
  def body
@@ -22,11 +22,11 @@ module Circuitry
22
22
  end
23
23
 
24
24
  def id
25
- raw['MessageId']
25
+ sqs_message.message_id
26
26
  end
27
27
 
28
28
  def receipt_handle
29
- raw['ReceiptHandle']
29
+ sqs_message.receipt_handle
30
30
  end
31
31
  end
32
32
  end
@@ -28,16 +28,12 @@ module Circuitry
28
28
  def publish(topic_name, object)
29
29
  raise ArgumentError.new('topic_name cannot be nil') if topic_name.nil?
30
30
  raise ArgumentError.new('object cannot be nil') if object.nil?
31
-
32
- unless can_publish?
33
- logger.warn('Circuitry unable to publish: AWS configuration is not set.')
34
- return
35
- end
31
+ raise PublishError.new('AWS configuration is not set') unless can_publish?
36
32
 
37
33
  process = -> do
38
34
  Timeout.timeout(timeout) do
39
35
  topic = TopicCreator.find_or_create(topic_name)
40
- sns.publish(topic.arn, object.to_json)
36
+ sns.publish(topic_arn: topic.arn, message: object.to_json)
41
37
  end
42
38
  end
43
39
 
@@ -1,10 +1,10 @@
1
- require 'fog/aws'
1
+ require 'aws-sdk'
2
2
 
3
3
  module Circuitry
4
4
  module Services
5
5
  module SNS
6
6
  def sns
7
- @sns ||= Fog::AWS::SNS.new(Circuitry.config.aws_options)
7
+ @sns ||= Aws::SNS::Client.new(Circuitry.config.aws_options)
8
8
  end
9
9
  end
10
10
  end
@@ -1,10 +1,10 @@
1
- require 'fog/aws'
1
+ require 'aws-sdk'
2
2
 
3
3
  module Circuitry
4
4
  module Services
5
5
  module SQS
6
6
  def sqs
7
- @sqs ||= Fog::AWS::SQS.new(Circuitry.config.aws_options)
7
+ @sqs ||= Aws::SQS::Client.new(Circuitry.config.aws_options)
8
8
  end
9
9
  end
10
10
  end
@@ -1,3 +1,4 @@
1
+ require 'retries'
1
2
  require 'timeout'
2
3
  require 'circuitry/concerns/async'
3
4
  require 'circuitry/services/sqs'
@@ -21,14 +22,7 @@ module Circuitry
21
22
  }.freeze
22
23
 
23
24
  CONNECTION_ERRORS = [
24
- Excon::Errors::Forbidden,
25
- ].freeze
26
-
27
- TEMPORARY_ERRORS = [
28
- Excon::Errors::InternalServerError,
29
- Excon::Errors::ServiceUnavailable,
30
- Excon::Errors::SocketError,
31
- Excon::Errors::Timeout,
25
+ Aws::SQS::Errors::ServiceError,
32
26
  ].freeze
33
27
 
34
28
  def initialize(queue, options = {})
@@ -36,30 +30,35 @@ module Circuitry
36
30
 
37
31
  options = DEFAULT_OPTIONS.merge(options)
38
32
 
33
+ self.subscribed = false
39
34
  self.queue = queue
40
35
  self.lock = options[:lock]
41
36
  self.async = options[:async]
42
37
  self.timeout = options[:timeout]
43
38
  self.wait_time = options[:wait_time]
44
39
  self.batch_size = options[:batch_size]
40
+
41
+ trap_signals
45
42
  end
46
43
 
47
44
  def subscribe(&block)
48
45
  raise ArgumentError.new('block required') if block.nil?
46
+ raise SubscribeError.new('AWS configuration is not set') unless can_subscribe?
49
47
 
50
- unless can_subscribe?
51
- logger.warn('Circuitry unable to subscribe: AWS configuration is not set.')
52
- return
53
- end
48
+ logger.info("Subscribing to queue: #{queue}")
54
49
 
55
- loop do
56
- begin
57
- receive_messages(&block)
58
- rescue *CONNECTION_ERRORS => e
59
- logger.error("Connection error to #{queue}: #{e}")
60
- raise SubscribeError.new(e)
61
- end
62
- end
50
+ self.subscribed = true
51
+ poll(&block)
52
+ self.subscribed = false
53
+
54
+ logger.info("Unsubscribed from queue: #{queue}")
55
+ rescue *CONNECTION_ERRORS => e
56
+ logger.error("Connection error to queue: #{queue}: #{e}")
57
+ raise SubscribeError.new(e)
58
+ end
59
+
60
+ def subscribed?
61
+ subscribed
63
62
  end
64
63
 
65
64
  def self.async_strategies
@@ -73,6 +72,7 @@ module Circuitry
73
72
  protected
74
73
 
75
74
  attr_writer :queue, :timeout, :wait_time, :batch_size
75
+ attr_accessor :subscribed
76
76
 
77
77
  def lock=(value)
78
78
  value = case value
@@ -87,19 +87,28 @@ module Circuitry
87
87
 
88
88
  private
89
89
 
90
- def receive_messages(&block)
91
- response = nil
90
+ def trap_signals
91
+ trap('SIGINT') do
92
+ if subscribed?
93
+ Thread.new { logger.info('Interrupt received, unsubscribing from queue...') }
94
+ self.subscribed = false
95
+ end
96
+ end
97
+ end
98
+
99
+ def poll(&block)
100
+ poller = Aws::SQS::QueuePoller.new(queue, client: sqs)
92
101
 
93
- begin
94
- response = sqs.receive_message(queue, 'MaxNumberOfMessages' => batch_size, 'WaitTimeSeconds' => wait_time)
95
- rescue *TEMPORARY_ERRORS => e
96
- logger.info("Temporary issue connecting to SQS: #{e.message}")
97
- return
102
+ poller.before_request do |_stats|
103
+ throw :stop_polling unless subscribed?
98
104
  end
99
105
 
100
- messages = response.body['Message']
101
- return if messages.empty?
106
+ poller.poll(max_number_of_messages: batch_size, wait_time_seconds: wait_time, skip_delete: true) do |messages|
107
+ process_messages(Array(messages), &block)
108
+ end
109
+ end
102
110
 
111
+ def process_messages(messages, &block)
103
112
  messages.each do |message|
104
113
  process = -> do
105
114
  process_message(message, &block)
@@ -144,7 +153,7 @@ module Circuitry
144
153
 
145
154
  def delete_message(message)
146
155
  logger.info("Removing message #{message.id} from queue")
147
- sqs.delete_message(queue, message.receipt_handle)
156
+ sqs.delete_message(queue_url: queue, receipt_handle: message.receipt_handle)
148
157
  end
149
158
 
150
159
  def logger
@@ -2,8 +2,6 @@ require 'circuitry/services/sns'
2
2
  require 'circuitry/topic'
3
3
 
4
4
  module Circuitry
5
- class TopicCreatorError < StandardError; end
6
-
7
5
  class TopicCreator
8
6
  include Services::SNS
9
7
 
@@ -20,9 +18,8 @@ module Circuitry
20
18
  def topic
21
19
  return @topic if defined?(@topic)
22
20
 
23
- response = sns.create_topic(topic_name)
24
- arn = response.body.fetch('TopicArn') { raise TopicCreatorError.new('No TopicArn returned from SNS') }
25
- @topic = Topic.new(arn)
21
+ response = sns.create_topic(name: topic_name)
22
+ @topic = Topic.new(response.topic_arn)
26
23
  end
27
24
  end
28
25
  end
@@ -1,3 +1,3 @@
1
1
  module Circuitry
2
- VERSION = '1.3.1'
2
+ VERSION = '1.4.0'
3
3
  end
metadata CHANGED
@@ -1,29 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: circuitry
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.1
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matt Huggins
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2015-11-06 00:00:00.000000000 Z
11
+ date: 2015-11-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: fog-aws
14
+ name: aws-sdk
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0.4'
19
+ version: '2'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0.4'
26
+ version: '2'
27
+ - !ruby/object:Gem::Dependency
28
+ name: retries
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 0.0.5
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 0.0.5
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: virtus
29
43
  requirement: !ruby/object:Gem::Requirement
@@ -178,6 +192,20 @@ dependencies:
178
192
  - - ">="
179
193
  - !ruby/object:Gem::Version
180
194
  version: '0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: connection_pool
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: '0'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: '0'
181
209
  description: Amazon SNS publishing and SQS queue processing.
182
210
  email:
183
211
  - matt.huggins@kapost.com
@@ -185,9 +213,9 @@ executables: []
185
213
  extensions: []
186
214
  extra_rdoc_files: []
187
215
  files:
188
- - ".codeclimate.yml"
189
216
  - ".gitignore"
190
217
  - ".rspec"
218
+ - CHANGELOG.md
191
219
  - Gemfile
192
220
  - LICENSE.txt
193
221
  - README.md
@@ -1,9 +0,0 @@
1
- ---
2
- engines:
3
- rubocop:
4
- enabled: true
5
- ratings:
6
- paths:
7
- - "**.rb"
8
- exclude_paths:
9
- - spec/**/*