carnivore-sqs 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/CONTRIBUTING.md +25 -0
- data/LICENSE +13 -0
- data/carnivore-sqs.gemspec +1 -0
- data/lib/carnivore-sqs/sqs.rb +27 -7
- data/lib/carnivore-sqs/version.rb +1 -3
- metadata +20 -23
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ed558475ba23fbf8a12d0c309972ee6576f4b417
|
4
|
+
data.tar.gz: e2033a5a84af8a75d40dadae088f689b2316009a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6f3c7036b47c9dea5393256900618f4a4a9aa5f8caaa43b21383680a6cf79845808d5dec692cecbaad08aac6f74fafc1e422cd906cebc9c41c933b9892974195
|
7
|
+
data.tar.gz: fdc46dbadd7f0cf9d2742dbfb8b674432c8ca41dae32526564e16463a5a14f9062dc43a007fbe1592e1a974f339ea1cf9a0ebf55e95616f6ac8c89d17f5caf09
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
# v0.1.4
|
2
|
+
* Make remote queue connection more resistant to failures
|
3
|
+
* Add better logging for connection setup and establishment
|
4
|
+
* Timeout should only be enabled when processing enabled
|
5
|
+
|
1
6
|
# v0.1.2
|
2
7
|
* Fix up queue formatting to prevent error on actor restarts
|
3
8
|
* Always dump transmit object if not string type
|
data/CONTRIBUTING.md
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# Contributing
|
2
|
+
|
3
|
+
## Branches
|
4
|
+
|
5
|
+
### `master` branch
|
6
|
+
|
7
|
+
The master branch is the current stable released version.
|
8
|
+
|
9
|
+
### `develop` branch
|
10
|
+
|
11
|
+
The develop branch is the current edge of development.
|
12
|
+
|
13
|
+
## Pull requests
|
14
|
+
|
15
|
+
* https://github.com/carnivore-rb/carnivore-sqs/pulls
|
16
|
+
|
17
|
+
Please base all pull requests of the `develop` branch. Merges to
|
18
|
+
`master` only occur through the `develop` branch. Pull requests
|
19
|
+
based on `master` will likely be cherry picked.
|
20
|
+
|
21
|
+
## Issues
|
22
|
+
|
23
|
+
Need to report an issue? Use the github issues:
|
24
|
+
|
25
|
+
* https://github.com/carnivore-rb/carnivore-sqs/issues
|
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2014 Chris Roberts
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/carnivore-sqs.gemspec
CHANGED
@@ -8,6 +8,7 @@ Gem::Specification.new do |s|
|
|
8
8
|
s.email = 'chrisroberts.code@gmail.com'
|
9
9
|
s.homepage = 'https://github.com/carnivore-rb/carnivore-sqs'
|
10
10
|
s.description = 'Carnivore SQS source'
|
11
|
+
s.license = 'Apache 2.0'
|
11
12
|
s.require_path = 'lib'
|
12
13
|
s.add_dependency 'carnivore', '>= 0.1.8'
|
13
14
|
s.add_dependency 'fog'
|
data/lib/carnivore-sqs/sqs.rb
CHANGED
@@ -28,9 +28,9 @@ module Carnivore
|
|
28
28
|
@processable_queues = Array(args[:processable_queues]).flatten.compact
|
29
29
|
end
|
30
30
|
@pause_time = args[:pause] || 5
|
31
|
-
@receive_timeout =
|
32
|
-
debug "
|
33
|
-
debug "
|
31
|
+
@receive_timeout = nil
|
32
|
+
debug "Setup for SQS source instance <#{name}> complete"
|
33
|
+
debug "Configured queues for handling: #{@queues.inspect}"
|
34
34
|
end
|
35
35
|
|
36
36
|
def format_queue(q)
|
@@ -47,16 +47,25 @@ module Carnivore
|
|
47
47
|
|
48
48
|
def connect
|
49
49
|
@fog = Fog::AWS::SQS.new(@connection_args)
|
50
|
+
debug "Connection for SQS source instance <#{name}> is now complete"
|
50
51
|
end
|
51
52
|
|
52
53
|
def receive(n=1)
|
54
|
+
set_receive_timeout!
|
53
55
|
count = 0
|
54
56
|
msgs = []
|
55
57
|
while(msgs.empty?)
|
56
58
|
msgs = []
|
57
59
|
@receive_timeout.reset
|
58
60
|
msgs = queues.map do |q|
|
59
|
-
|
61
|
+
begin
|
62
|
+
m = @fog.receive_message(q, 'MaxNumberOfMessages' => n).body['Message']
|
63
|
+
rescue Excon::Errors::Error => e
|
64
|
+
error "SQS received an unexpected error. Pausing and running retry (#{e.class}: #{e})"
|
65
|
+
debug "SQS ERROR TRACE: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
66
|
+
sleep Carnivore::Config.get(:carnivore, :sqs, :retry_wait) || 5
|
67
|
+
retry
|
68
|
+
end
|
60
69
|
m.map! do |msg|
|
61
70
|
msg.merge('SourceQueue' => q)
|
62
71
|
end
|
@@ -78,9 +87,16 @@ module Carnivore
|
|
78
87
|
end
|
79
88
|
|
80
89
|
def transmit(message, original=nil)
|
81
|
-
|
82
|
-
|
83
|
-
|
90
|
+
begin
|
91
|
+
queue = determine_queue(original)
|
92
|
+
message = JSON.dump(message) unless message.is_a?(String)
|
93
|
+
@fog.send_message(queue, message)
|
94
|
+
rescue Excon::Errors::Error => e
|
95
|
+
error "SQS transmission received an unexpected error. Pausing and running retry (#{e.class}: #{e})"
|
96
|
+
debug "SQS ERROR TRACE: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
97
|
+
sleep Carnivore::Config.get(:carnivore, :sqs, :retry_wait) || 5
|
98
|
+
retry
|
99
|
+
end
|
84
100
|
end
|
85
101
|
|
86
102
|
def confirm(message)
|
@@ -143,6 +159,10 @@ module Carnivore
|
|
143
159
|
m
|
144
160
|
end
|
145
161
|
|
162
|
+
def set_receive_timeout!
|
163
|
+
@receive_timeout ||= after(args[:receive_timeout] || 30){ connect }
|
164
|
+
end
|
165
|
+
|
146
166
|
end
|
147
167
|
end
|
148
168
|
end
|
metadata
CHANGED
@@ -1,46 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: carnivore-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
5
|
-
prerelease:
|
4
|
+
version: 0.1.4
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Chris Roberts
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2014-
|
11
|
+
date: 2014-08-14 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: carnivore
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: 0.1.8
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: 0.1.8
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: fog
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ">="
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ">="
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
description: Carnivore SQS source
|
@@ -49,35 +44,37 @@ executables: []
|
|
49
44
|
extensions: []
|
50
45
|
extra_rdoc_files: []
|
51
46
|
files:
|
52
|
-
- lib/carnivore-sqs/version.rb
|
53
|
-
- lib/carnivore-sqs/sqs.rb
|
54
|
-
- lib/carnivore-sqs.rb
|
55
|
-
- carnivore-sqs.gemspec
|
56
|
-
- README.md
|
57
47
|
- CHANGELOG.md
|
48
|
+
- CONTRIBUTING.md
|
49
|
+
- LICENSE
|
50
|
+
- README.md
|
51
|
+
- carnivore-sqs.gemspec
|
52
|
+
- lib/carnivore-sqs.rb
|
53
|
+
- lib/carnivore-sqs/sqs.rb
|
54
|
+
- lib/carnivore-sqs/version.rb
|
58
55
|
homepage: https://github.com/carnivore-rb/carnivore-sqs
|
59
|
-
licenses:
|
56
|
+
licenses:
|
57
|
+
- Apache 2.0
|
58
|
+
metadata: {}
|
60
59
|
post_install_message:
|
61
60
|
rdoc_options: []
|
62
61
|
require_paths:
|
63
62
|
- lib
|
64
63
|
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
64
|
requirements:
|
67
|
-
- -
|
65
|
+
- - ">="
|
68
66
|
- !ruby/object:Gem::Version
|
69
67
|
version: '0'
|
70
68
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
71
|
-
none: false
|
72
69
|
requirements:
|
73
|
-
- -
|
70
|
+
- - ">="
|
74
71
|
- !ruby/object:Gem::Version
|
75
72
|
version: '0'
|
76
73
|
requirements: []
|
77
74
|
rubyforge_project:
|
78
|
-
rubygems_version:
|
75
|
+
rubygems_version: 2.2.2
|
79
76
|
signing_key:
|
80
|
-
specification_version:
|
77
|
+
specification_version: 4
|
81
78
|
summary: Message processing helper
|
82
79
|
test_files: []
|
83
80
|
has_rdoc:
|