circuitry 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +40 -0
- data/README.md +4 -3
- data/circuitry.gemspec +3 -1
- data/lib/circuitry/configuration.rb +3 -3
- data/lib/circuitry/message.rb +6 -6
- data/lib/circuitry/publisher.rb +2 -6
- data/lib/circuitry/services/sns.rb +2 -2
- data/lib/circuitry/services/sqs.rb +2 -2
- data/lib/circuitry/subscriber.rb +39 -30
- data/lib/circuitry/topic_creator.rb +2 -5
- data/lib/circuitry/version.rb +1 -1
- metadata +34 -6
- data/.codeclimate.yml +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aacd53812987507821869c3aee366956a7f824ba
|
4
|
+
data.tar.gz: a5c069dc6b82c398b89d06dd3aa826fc8191869e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be97c0b2b282f712eefe048a4da9c412f7e85df3921e89ca5aa9829bdf1a3f4700dfeb60812e131001790da5e2948ecd9c66e419f2c09b6a04f6969b145985c1
|
7
|
+
data.tar.gz: 0ae5a71aac398cdba9bfa9a3a8771cc0eb6e6455088f7fb9459e1941f41916b607456aa7d013e09ffe5f09e3f68290f7983b96005d9f26234acc44379a4ccc73
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
## Circuitry 1.4.0 (Nov 15, 2015)
|
2
|
+
|
3
|
+
* Replace [fog-aws](https://github.com/fog/fog-aws) with
|
4
|
+
[aws-sdk](https://github.com/aws/aws-sdk-ruby). *Matt Huggins*
|
5
|
+
* Fix long polling for subscriber. *Matt Huggins*
|
6
|
+
* Retry message deletion if it fails after successful message processing. *Matt Huggins*
|
7
|
+
|
8
|
+
## Circuitry 1.3.1 (Nov 6, 2015)
|
9
|
+
|
10
|
+
* Implement redis connection pooling for redis lock strategy. *Matt Huggins*
|
11
|
+
|
12
|
+
## Circuitry 1.3.0 (Oct 9, 2015)
|
13
|
+
|
14
|
+
* Implement thread exit and fork exit hooks. *Matt Huggins*
|
15
|
+
|
16
|
+
## Circuitry 1.2.3 (Aug 26, 2015)
|
17
|
+
|
18
|
+
* Treat connection resets as temporary service errors. *Matt Huggins*
|
19
|
+
|
20
|
+
## Circuitry 1.2.2 (Aug 13, 2015)
|
21
|
+
|
22
|
+
* Ignore temporary service errors. *Matt Huggins*
|
23
|
+
|
24
|
+
## Circuitry 1.2.1 (Jul 29, 2015)
|
25
|
+
|
26
|
+
* Unlock soft locks from messages that were unsuccessfully processed. *Matt Huggins*
|
27
|
+
|
28
|
+
## Circuitry 1.2.0 (Jul 20, 2015)
|
29
|
+
|
30
|
+
* Implement lock strategies to prevent duplicate message processing. *Matt Huggins*
|
31
|
+
* Implement timeout for message processing on both subscriber and publisher. *Matt Huggins*
|
32
|
+
|
33
|
+
## Circuitry 1.1.0 (Jul 8, 2015)
|
34
|
+
|
35
|
+
* Permit forking, threading, and batching async strategies for both subscriber and publisher.
|
36
|
+
*Matt Huggins*
|
37
|
+
|
38
|
+
## Circuitry 1.0.0 (Jun 25, 2015)
|
39
|
+
|
40
|
+
* Initial release. *Matt Huggins*
|
data/README.md
CHANGED
@@ -410,6 +410,7 @@ and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
410
410
|
|
411
411
|
1. Fork it ( https://github.com/kapost/circuitry/fork )
|
412
412
|
2. Create your feature branch (`git checkout -b my-new-feature`)
|
413
|
-
3.
|
414
|
-
4.
|
415
|
-
5.
|
413
|
+
3. Update the changelog
|
414
|
+
4. Commit your changes (`git commit -am 'Add some feature'`)
|
415
|
+
5. Push to the branch (`git push origin my-new-feature`)
|
416
|
+
6. Create a new Pull Request
|
data/circuitry.gemspec
CHANGED
@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
|
22
|
-
spec.add_dependency '
|
22
|
+
spec.add_dependency 'aws-sdk', '~> 2'
|
23
|
+
spec.add_dependency 'retries', '~> 0.0.5'
|
23
24
|
spec.add_dependency 'virtus', '~> 1.0'
|
24
25
|
|
25
26
|
spec.add_development_dependency 'pry-byebug'
|
@@ -32,4 +33,5 @@ Gem::Specification.new do |spec|
|
|
32
33
|
spec.add_development_dependency 'mock_redis'
|
33
34
|
spec.add_development_dependency 'dalli'
|
34
35
|
spec.add_development_dependency 'memcache_mock'
|
36
|
+
spec.add_development_dependency 'connection_pool'
|
35
37
|
end
|
data/lib/circuitry/message.rb
CHANGED
@@ -3,14 +3,14 @@ require 'circuitry/topic'
|
|
3
3
|
|
4
4
|
module Circuitry
|
5
5
|
class Message
|
6
|
-
attr_reader :
|
6
|
+
attr_reader :sqs_message
|
7
7
|
|
8
|
-
def initialize(
|
9
|
-
@
|
8
|
+
def initialize(sqs_message)
|
9
|
+
@sqs_message = sqs_message
|
10
10
|
end
|
11
11
|
|
12
12
|
def context
|
13
|
-
@context ||= JSON.parse(
|
13
|
+
@context ||= JSON.parse(sqs_message.body)
|
14
14
|
end
|
15
15
|
|
16
16
|
def body
|
@@ -22,11 +22,11 @@ module Circuitry
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def id
|
25
|
-
|
25
|
+
sqs_message.message_id
|
26
26
|
end
|
27
27
|
|
28
28
|
def receipt_handle
|
29
|
-
|
29
|
+
sqs_message.receipt_handle
|
30
30
|
end
|
31
31
|
end
|
32
32
|
end
|
data/lib/circuitry/publisher.rb
CHANGED
@@ -28,16 +28,12 @@ module Circuitry
|
|
28
28
|
def publish(topic_name, object)
|
29
29
|
raise ArgumentError.new('topic_name cannot be nil') if topic_name.nil?
|
30
30
|
raise ArgumentError.new('object cannot be nil') if object.nil?
|
31
|
-
|
32
|
-
unless can_publish?
|
33
|
-
logger.warn('Circuitry unable to publish: AWS configuration is not set.')
|
34
|
-
return
|
35
|
-
end
|
31
|
+
raise PublishError.new('AWS configuration is not set') unless can_publish?
|
36
32
|
|
37
33
|
process = -> do
|
38
34
|
Timeout.timeout(timeout) do
|
39
35
|
topic = TopicCreator.find_or_create(topic_name)
|
40
|
-
sns.publish(topic.arn, object.to_json)
|
36
|
+
sns.publish(topic_arn: topic.arn, message: object.to_json)
|
41
37
|
end
|
42
38
|
end
|
43
39
|
|
data/lib/circuitry/subscriber.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'retries'
|
1
2
|
require 'timeout'
|
2
3
|
require 'circuitry/concerns/async'
|
3
4
|
require 'circuitry/services/sqs'
|
@@ -21,14 +22,7 @@ module Circuitry
|
|
21
22
|
}.freeze
|
22
23
|
|
23
24
|
CONNECTION_ERRORS = [
|
24
|
-
|
25
|
-
].freeze
|
26
|
-
|
27
|
-
TEMPORARY_ERRORS = [
|
28
|
-
Excon::Errors::InternalServerError,
|
29
|
-
Excon::Errors::ServiceUnavailable,
|
30
|
-
Excon::Errors::SocketError,
|
31
|
-
Excon::Errors::Timeout,
|
25
|
+
Aws::SQS::Errors::ServiceError,
|
32
26
|
].freeze
|
33
27
|
|
34
28
|
def initialize(queue, options = {})
|
@@ -36,30 +30,35 @@ module Circuitry
|
|
36
30
|
|
37
31
|
options = DEFAULT_OPTIONS.merge(options)
|
38
32
|
|
33
|
+
self.subscribed = false
|
39
34
|
self.queue = queue
|
40
35
|
self.lock = options[:lock]
|
41
36
|
self.async = options[:async]
|
42
37
|
self.timeout = options[:timeout]
|
43
38
|
self.wait_time = options[:wait_time]
|
44
39
|
self.batch_size = options[:batch_size]
|
40
|
+
|
41
|
+
trap_signals
|
45
42
|
end
|
46
43
|
|
47
44
|
def subscribe(&block)
|
48
45
|
raise ArgumentError.new('block required') if block.nil?
|
46
|
+
raise SubscribeError.new('AWS configuration is not set') unless can_subscribe?
|
49
47
|
|
50
|
-
|
51
|
-
logger.warn('Circuitry unable to subscribe: AWS configuration is not set.')
|
52
|
-
return
|
53
|
-
end
|
48
|
+
logger.info("Subscribing to queue: #{queue}")
|
54
49
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
50
|
+
self.subscribed = true
|
51
|
+
poll(&block)
|
52
|
+
self.subscribed = false
|
53
|
+
|
54
|
+
logger.info("Unsubscribed from queue: #{queue}")
|
55
|
+
rescue *CONNECTION_ERRORS => e
|
56
|
+
logger.error("Connection error to queue: #{queue}: #{e}")
|
57
|
+
raise SubscribeError.new(e)
|
58
|
+
end
|
59
|
+
|
60
|
+
def subscribed?
|
61
|
+
subscribed
|
63
62
|
end
|
64
63
|
|
65
64
|
def self.async_strategies
|
@@ -73,6 +72,7 @@ module Circuitry
|
|
73
72
|
protected
|
74
73
|
|
75
74
|
attr_writer :queue, :timeout, :wait_time, :batch_size
|
75
|
+
attr_accessor :subscribed
|
76
76
|
|
77
77
|
def lock=(value)
|
78
78
|
value = case value
|
@@ -87,19 +87,28 @@ module Circuitry
|
|
87
87
|
|
88
88
|
private
|
89
89
|
|
90
|
-
def
|
91
|
-
|
90
|
+
def trap_signals
|
91
|
+
trap('SIGINT') do
|
92
|
+
if subscribed?
|
93
|
+
Thread.new { logger.info('Interrupt received, unsubscribing from queue...') }
|
94
|
+
self.subscribed = false
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def poll(&block)
|
100
|
+
poller = Aws::SQS::QueuePoller.new(queue, client: sqs)
|
92
101
|
|
93
|
-
|
94
|
-
|
95
|
-
rescue *TEMPORARY_ERRORS => e
|
96
|
-
logger.info("Temporary issue connecting to SQS: #{e.message}")
|
97
|
-
return
|
102
|
+
poller.before_request do |_stats|
|
103
|
+
throw :stop_polling unless subscribed?
|
98
104
|
end
|
99
105
|
|
100
|
-
|
101
|
-
|
106
|
+
poller.poll(max_number_of_messages: batch_size, wait_time_seconds: wait_time, skip_delete: true) do |messages|
|
107
|
+
process_messages(Array(messages), &block)
|
108
|
+
end
|
109
|
+
end
|
102
110
|
|
111
|
+
def process_messages(messages, &block)
|
103
112
|
messages.each do |message|
|
104
113
|
process = -> do
|
105
114
|
process_message(message, &block)
|
@@ -144,7 +153,7 @@ module Circuitry
|
|
144
153
|
|
145
154
|
def delete_message(message)
|
146
155
|
logger.info("Removing message #{message.id} from queue")
|
147
|
-
sqs.delete_message(queue, message.receipt_handle)
|
156
|
+
sqs.delete_message(queue_url: queue, receipt_handle: message.receipt_handle)
|
148
157
|
end
|
149
158
|
|
150
159
|
def logger
|
@@ -2,8 +2,6 @@ require 'circuitry/services/sns'
|
|
2
2
|
require 'circuitry/topic'
|
3
3
|
|
4
4
|
module Circuitry
|
5
|
-
class TopicCreatorError < StandardError; end
|
6
|
-
|
7
5
|
class TopicCreator
|
8
6
|
include Services::SNS
|
9
7
|
|
@@ -20,9 +18,8 @@ module Circuitry
|
|
20
18
|
def topic
|
21
19
|
return @topic if defined?(@topic)
|
22
20
|
|
23
|
-
response = sns.create_topic(topic_name)
|
24
|
-
|
25
|
-
@topic = Topic.new(arn)
|
21
|
+
response = sns.create_topic(name: topic_name)
|
22
|
+
@topic = Topic.new(response.topic_arn)
|
26
23
|
end
|
27
24
|
end
|
28
25
|
end
|
data/lib/circuitry/version.rb
CHANGED
metadata
CHANGED
@@ -1,29 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: circuitry
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matt Huggins
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-11-
|
11
|
+
date: 2015-11-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: aws-sdk
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: retries
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.0.5
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.0.5
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: virtus
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,6 +192,20 @@ dependencies:
|
|
178
192
|
- - ">="
|
179
193
|
- !ruby/object:Gem::Version
|
180
194
|
version: '0'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: connection_pool
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - ">="
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0'
|
181
209
|
description: Amazon SNS publishing and SQS queue processing.
|
182
210
|
email:
|
183
211
|
- matt.huggins@kapost.com
|
@@ -185,9 +213,9 @@ executables: []
|
|
185
213
|
extensions: []
|
186
214
|
extra_rdoc_files: []
|
187
215
|
files:
|
188
|
-
- ".codeclimate.yml"
|
189
216
|
- ".gitignore"
|
190
217
|
- ".rspec"
|
218
|
+
- CHANGELOG.md
|
191
219
|
- Gemfile
|
192
220
|
- LICENSE.txt
|
193
221
|
- README.md
|