sqs-job 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: e08b70c3678927599b80b3e4ac69146633dbdcef
4
+ data.tar.gz: 8e460f4f8bf5627d36c65a52d7a8469d5c6d89f2
5
+ SHA512:
6
+ metadata.gz: 394347d447b86f505368070c0e1824c21114d9d47a044e1d6ac681210a2132720578645a079cb762ecd73f7b2689f6f8f9c23fbbfd24f6a48593de8e9b5a4608
7
+ data.tar.gz: 2531d606b29ad4a615561b1f34f0c0fe5bd2aac5c9683e0ab9ebb787e41089e8eb506a562e250baa97c150ebcfc2b8497ba348d37e9309b21444f7dd4b997191
@@ -0,0 +1,25 @@
1
+ features/reports
2
+ spec/reports
3
+ policy.json
4
+ *.gem
5
+ *.rbc
6
+ .bundle
7
+ .config
8
+ .yardoc
9
+ Gemfile.lock
10
+ InstalledFiles
11
+ _yardoc
12
+ coverage
13
+ doc/
14
+ lib/bundler/man
15
+ pkg
16
+ rdoc
17
+ spec/reports
18
+ test/tmp
19
+ test/version_tmp
20
+ tmp
21
+ *.bundle
22
+ *.so
23
+ *.o
24
+ *.a
25
+ mkmf.log
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <projectDescription>
3
+ <name>sqs-job</name>
4
+ <comment></comment>
5
+ <projects>
6
+ </projects>
7
+ <buildSpec>
8
+ <buildCommand>
9
+ <name>com.aptana.ide.core.unifiedBuilder</name>
10
+ <arguments>
11
+ </arguments>
12
+ </buildCommand>
13
+ </buildSpec>
14
+ <natures>
15
+ <nature>com.aptana.ruby.core.rubynature</nature>
16
+ <nature>com.aptana.projects.webnature</nature>
17
+ </natures>
18
+ </projectDescription>
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ #ruby-gemset=sqs-job
4
+
5
+ # Specify your gem's dependencies in sqs-job.gemspec
6
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Kevin Gilpin
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,38 @@
1
+ # SQS::Job
2
+
3
+ Simple job processor which uses SQS.
4
+
5
+ Here's an interesting description of job processing using SQS, which isn't actually a spec of this code,
6
+ but is nicely related:
7
+
8
+ http://mauricio.github.io/2014/09/01/make-the-most-of-sqs.html
9
+
10
+ ## Installation
11
+
12
+ Add this line to your application's Gemfile:
13
+
14
+ gem 'sqs-job'
15
+
16
+ And then execute:
17
+
18
+ $ bundle
19
+
20
+ Or install it yourself as:
21
+
22
+ $ gem install sqs-job
23
+
24
+ # Cucumber
25
+
26
+ Populate aws.secrets with `aws_access_key_id` and `aws_secret_access_key`. Then:
27
+
28
+ $ conjur env run -c aws.secrets -- conjur policy load -c policy.json cucumber-policy.rb
29
+ $ conjur env run -c aws.secrets -- env POLICY_FILE=policy.json rake provision
30
+ $ cucumber
31
+
32
+ ## Contributing
33
+
34
+ 1. Fork it ( https://github.com/[my-github-username]/sqs-job/fork )
35
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
36
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
37
+ 4. Push to the branch (`git push origin my-new-feature`)
38
+ 5. Create a new Pull Request
@@ -0,0 +1,71 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ $LOAD_PATH.unshift 'lib'
4
+
5
+ require 'ci/reporter/rake/rspec'
6
+ require 'ci/reporter/rake/cucumber'
7
+ require 'cucumber'
8
+ require 'cucumber/rake/task'
9
+ require 'rspec/core/rake_task'
10
+
11
+ RSpec::Core::RakeTask.new :spec
12
+ Cucumber::Rake::Task.new :features
13
+
14
+ task :jenkins => ['ci:setup:rspec', :spec, 'ci:setup:cucumber_report_cleanup'] do
15
+ Cucumber::Rake::Task.new do |t|
16
+ t.cucumber_opts = "--format CI::Reporter::Cucumber"
17
+ end.runner.run
18
+ File.write('build_number', ENV['BUILD_NUMBER']) if ENV['BUILD_NUMBER']
19
+ end
20
+
21
+ desc 'Creates resources and loads access credentials into Conjur variables. Pre-requisite to cucumber tests.'
22
+ task :provision do
23
+ [ 'POLICY_FILE', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY' ].each do |v|
24
+ ENV[v] or raise "#{v} is a required environment variable"
25
+ end
26
+
27
+ require 'json'
28
+
29
+ policy = JSON.parse(File.read(ENV['POLICY_FILE']))
30
+ ENV['CONJUR_POLICY_ID'] = policy_id = policy['policy']
31
+
32
+ require 'conjur/cli'
33
+ require 'sqs/job'
34
+ require 'aws-sdk'
35
+
36
+ Conjur::Config.load
37
+ Conjur::Config.apply
38
+
39
+ conjur = Conjur::Authn.connect
40
+ iam = AWS::IAM.new
41
+ sqs = AWS::SQS.new
42
+
43
+ queue_name = [ policy_id.gsub(/[^a-zA-Z0-9-]/, '-'), 'job-queue' ].join('-')
44
+ user_name = [ 'sys', policy_id.gsub(/[^a-zA-Z0-9-]/, '_'), 'alice' ].join('_')
45
+
46
+ job_provisioner = SQS::Job::Provisoner.new(conjur, policy_id)
47
+
48
+ $stderr.puts "Creating signing key"
49
+ job_provisioner.create_signing_key 'jobs'
50
+
51
+ sqs_queue = begin
52
+ sqs.queues.named(queue_name)
53
+ rescue AWS::SQS::Errors::NonExistentQueue
54
+ $stderr.puts "Creating SQS queue #{queue_name}"
55
+ sqs.queues.create(queue_name)
56
+ end
57
+
58
+ user = iam.users[user_name]
59
+ unless user.exists?
60
+ $stderr.puts "Creating IAM user #{user_name}"
61
+ user = iam.users.create(user_name)
62
+ end
63
+
64
+ job_provisioner.permit_queue_send user, sqs_queue
65
+ job_provisioner.permit_queue_receive user, sqs_queue
66
+
67
+ access_key = user.access_keys.create
68
+ $stderr.puts "Saving access_key_id and secret_access_key"
69
+ conjur.variable([ policy_id, 'aws/access_key_id'].join('/')).add_value access_key.id
70
+ conjur.variable([ policy_id, 'aws/secret_access_key'].join('/')).add_value access_key.secret
71
+ end
@@ -0,0 +1,16 @@
1
+ $LOAD_PATH.unshift 'lib'
2
+ require 'sqs/job'
3
+
4
+ policy "sqs-job-cucumber-1.0" do
5
+ create_signing_key_variables 'jobs'
6
+
7
+ aws_credentials = [
8
+ variable("aws/access_key_id"),
9
+ variable("aws/secret_access_key")
10
+ ]
11
+
12
+ user "alice" do
13
+ can_submit_job 'jobs', aws_credentials
14
+ can_process_job 'jobs', aws_credentials
15
+ end
16
+ end
@@ -0,0 +1,5 @@
1
+ Feature: Successfully send a message
2
+
3
+ Scenario: A sent message can be received and processed
4
+ When I send a message
5
+ Then the message is processed
@@ -0,0 +1,7 @@
1
+ module SQS::Job::Message
2
+ class TestMessage < Base
3
+ def invoke!
4
+ $messages_received << TestMessage
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,8 @@
1
+ When(/^I send a message$/) do
2
+ SQS::Job.send_message $queue, "test_message", {}
3
+ sleep 2
4
+ end
5
+
6
+ Then(/^the message is processed$/) do
7
+ expect($messages_received).to eq([ SQS::Job::Message::TestMessage ])
8
+ end
@@ -0,0 +1,43 @@
1
+ $LOAD_PATH.unshift 'features'
2
+
3
+ require 'conjur/cli'
4
+ require 'sqs/job'
5
+ require 'slosilo'
6
+
7
+ Conjur::Config.load
8
+ Conjur::Config.apply
9
+ conjur = Conjur::Authn.connect
10
+
11
+ policy = JSON.parse(File.read('policy.json'))
12
+ policy_id = policy['policy']
13
+
14
+ ENV['AWS_ACCESS_KEY_ID'] = conjur.variable([ policy_id, 'aws/access_key_id' ].join('/')).value
15
+ ENV['AWS_SECRET_ACCESS_KEY'] = conjur.variable([ policy_id, 'aws/secret_access_key' ].join('/')).value
16
+
17
+ signing_key_id = SQS::Job::Policy.private_variable_name([ policy_id, 'jobs' ].join('/'))
18
+ queue_name = [ policy_id.gsub(/[^a-zA-Z0-9-]/, '-'), 'job-queue' ].join('-')
19
+
20
+ require 'aws-sdk'
21
+ sqs ||= AWS::SQS::new
22
+ $queue = sqs.queues.named(queue_name)
23
+
24
+ SQS::Job.signing_keys = [ Slosilo::Key.new(conjur.variable(signing_key_id).value) ]
25
+
26
+ $messages_received = []
27
+ $worker_thread = nil
28
+
29
+ Before do
30
+ $messages_received.clear
31
+ $worker_thread = Thread.new do
32
+ begin
33
+ SQS::Job::Worker.new($queue).run
34
+ rescue
35
+ $stderr.puts $!
36
+ raise $!
37
+ end
38
+ end
39
+ end
40
+
41
+ After do
42
+ $worker_thread.kill
43
+ end
@@ -0,0 +1,65 @@
1
+ require "sqs/job/version"
2
+ require 'logger'
3
+ require 'active_support'
4
+ require 'active_support/core_ext'
5
+ require 'json'
6
+
7
+ module SQS
8
+ module Job
9
+ class << self
10
+ def send_message queue, type, parameters
11
+ require 'base64'
12
+
13
+ message = {
14
+ "type" => type,
15
+ "params" => parameters
16
+ }.to_json
17
+ signature = signing_keys.last.sign message
18
+ fingerprint = signing_keys.last.fingerprint
19
+ message_attributes = {
20
+ "signature" => {
21
+ "string_value" => Base64.strict_encode64(signature),
22
+ "data_type" => "String",
23
+ },
24
+ "key_fingerprint" => {
25
+ "string_value" => fingerprint,
26
+ "data_type" => "String"
27
+ }
28
+ }
29
+ queue.send_message message, message_attributes: message_attributes
30
+ end
31
+
32
+ def signature_valid? message, fingerprint, signature
33
+ !signing_keys.find do |k|
34
+ k.fingerprint == fingerprint && k.verify_signature(message, signature)
35
+ end.nil?
36
+ end
37
+
38
+ def min_threads
39
+ ENV['SQS_JOB_MIN_THREADS'] || 1
40
+ end
41
+
42
+ def max_threads
43
+ ENV['SQS_JOB_MAX_THREADS'] || 10
44
+ end
45
+
46
+ def signing_keys=(keys); @signing_keys = keys; end
47
+ def signing_keys; @signing_keys or raise "No signing keys are configured"; end
48
+
49
+ def logger=(logger); @logger = logger; end
50
+ def logger
51
+ @logger ||= Logger.new(STDERR)
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ require 'sqs/job/exceptions'
58
+ require 'sqs/job/worker'
59
+ require 'sqs/job/handler'
60
+ require 'sqs/job/message/base'
61
+ require 'sqs/job/provisioner'
62
+
63
+ if defined?(Conjur)
64
+ require 'sqs/job/policy'
65
+ end
@@ -0,0 +1,17 @@
1
+ class UnrecoverableException < RuntimeError
2
+ end
3
+
4
+ class InvalidMessageException < UnrecoverableException
5
+ def initialize(message)
6
+ super message.errors.full_messages.join(', ')
7
+ end
8
+ end
9
+
10
+ class MissingTypeException < UnrecoverableException
11
+ end
12
+
13
+ class SignatureInvalidException < UnrecoverableException
14
+ end
15
+
16
+ class UnrecognizedMessageTypeException < UnrecoverableException
17
+ end
@@ -0,0 +1,63 @@
1
+ module SQS::Job
2
+ # One Handler instance is created per SQS message
3
+ # received, and is responsible for processing it
4
+ # by instantiating a Message::Base subclass and
5
+ # calling its #invoke! method. The "main" method
6
+ # is #run!.
7
+ #
8
+ # Messages have the following structure:
9
+ # {type: 'create', params: { ... }}
10
+ # Where params is optional (for example, keepalive messages
11
+ # might not have params).
12
+ #
13
+ # The message class is loaded by requiring 'vm2/message/#{type}'
14
+ # constantizing it in the normal way, and passing the params hash
15
+ # to #new.
16
+ class Handler
17
+ def initialize sqs_message
18
+ @sqs_message = sqs_message
19
+ end
20
+
21
+ # Run this handler
22
+ def run!
23
+ require 'base64'
24
+
25
+ fingerprint = @sqs_message.message_attributes['key_fingerprint'][:string_value]
26
+ signature = Base64.strict_decode64(@sqs_message.message_attributes['signature'][:string_value])
27
+
28
+ raise SignatureInvalidException unless SQS::Job.signature_valid? @sqs_message.body, fingerprint, signature
29
+
30
+ msg = JSON.parse(@sqs_message.body)
31
+ raise MissingTypeException unless (type = msg['type'])
32
+
33
+ klass = message_class type
34
+ SQS::Job.logger.info "Received message #{klass}"
35
+
36
+ message = klass.new(msg['params'] || {})
37
+ raise InvalidMessageException, message unless message.valid?
38
+ message.invoke!
39
+ end
40
+
41
+ private
42
+
43
+ def message_class type
44
+ # This seems simpler than trying to fake abstract classes in ruby
45
+ raise UnrecognizedMessageTypeException, type if type == 'base'
46
+
47
+ # This would be the place to implement a message whitelist
48
+ message_file = "sqs/job/message/#{type}"
49
+
50
+ # We might do some caching here at some point...the only reason
51
+ # I'm not adding it now is that the multithreaded environment makes
52
+ # it slightly less than a freebie
53
+ begin
54
+ require message_file
55
+ rescue LoadError
56
+ SQS::Job.logger.info $!
57
+ raise UnrecognizedMessageTypeException, type
58
+ end
59
+
60
+ message_file.gsub(/^sqs/, 'SQS').classify.constantize
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,35 @@
1
+ require 'validatable'
2
+
3
+ module SQS::Job
4
+ module Message
5
+ # Base class for messages. A Message::Base subclass
6
+ # must implement an #invoke! method which performs the
7
+ # appropriate operation.
8
+ class Base
9
+ include Validatable
10
+
11
+ def initialize params
12
+ # We allow messages with no params field
13
+ @params = (params || {}).symbolize_keys.freeze
14
+ end
15
+
16
+ # Parameters for the message. This Hash is frozen.
17
+ attr_reader :params
18
+
19
+ # Id of this message. Used to send replies.
20
+ attr_reader :message_id
21
+
22
+ # Get a parameter or raise an exception if it's not present
23
+ # @param name [Symbol,String] the param name
24
+ def param! name
25
+ params[name.to_sym] or raise "Missing parameter #{name}"
26
+ end
27
+
28
+ # Type field for this message. Used primarily when sending
29
+ # replies.
30
+ def type
31
+ self.class.name.split('::')[-1].underscore
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,51 @@
1
+ module SQS::Job
2
+ module Policy
3
+ def create_signing_key_variables queue_name
4
+ options = {
5
+ 'mime_type' => 'application/x-pem-file'
6
+ }
7
+ public_key = variable(SQS::Job::Policy.public_variable_name(queue_name), options).tap do |v|
8
+ v.resource.annotations['kind'] = "RSA public key"
9
+ end
10
+ private_key = variable(SQS::Job::Policy.private_variable_name(queue_name), options).tap do |v|
11
+ v.resource.annotations['kind'] = "RSA private key"
12
+ end
13
+ [ public_key, private_key ].tap do |vars|
14
+ vars.each do |var|
15
+ options.each do |k,v|
16
+ var.resource.annotations[k] = v
17
+ end
18
+ var.resource.annotations['facility'] = 'sqs/job'
19
+ end
20
+ end
21
+ end
22
+
23
+ def can_submit_job queue_name, aws_credentials
24
+ can "execute", variable(SQS::Job::Policy.public_variable_name(queue_name))
25
+ aws_credentials.each do |var|
26
+ can "execute", var
27
+ end
28
+ end
29
+
30
+ def can_process_job queue_name, aws_credentials
31
+ can "execute", variable(SQS::Job::Policy.private_variable_name(queue_name))
32
+ aws_credentials.each do |var|
33
+ can "execute", var
34
+ end
35
+ end
36
+
37
+ class << self
38
+ def public_variable_name queue_name
39
+ [ queue_name, "signing-key/public" ].join('/')
40
+ end
41
+
42
+ def private_variable_name queue_name
43
+ [ queue_name, "signing-key/private" ].join('/')
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ Conjur::DSL::Runner.module_eval do
50
+ include SQS::Job::Policy
51
+ end
@@ -0,0 +1,46 @@
1
+ module SQS::Job
2
+ Provisoner = Struct.new(:conjur, :policy_id) do
3
+ # Create a signing key (Slosilo::Key) and store it in +variable_name+/public and +variable_name+/private.
4
+ def create_signing_key queue_name
5
+ require 'slosilo'
6
+ signing_key = Slosilo::Key.new
7
+ conjur.variable([ policy_id, SQS::Job::Policy.public_variable_name(queue_name) ].join('/')).add_value signing_key.key.public_key.to_pem
8
+ conjur.variable([ policy_id, SQS::Job::Policy.private_variable_name(queue_name) ].join('/')).add_value signing_key.key.to_pem
9
+ nil
10
+ end
11
+
12
+ # Configure a user policy to send to the sqs_queue.
13
+ def permit_queue_send user, sqs_queue
14
+ user.policies['send_to_queue'] = JSON.pretty_generate({
15
+ "Statement" => [
16
+ "Effect" => "Allow",
17
+ "Action" => [ "sqs:SendMessage" ],
18
+ "Resource" => [ sqs_queue.arn ]
19
+ ]})
20
+ user.policies['info'] = info_policy
21
+ end
22
+
23
+ # Configure a user policy to receive from the sqs_queue.
24
+ def permit_queue_receive user, sqs_queue
25
+ user.policies['receive_from_queue'] = JSON.pretty_generate({
26
+ "Statement" => [
27
+ "Effect" => "Allow",
28
+ "Action" => [ "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:ChangeMessageVisibility" ],
29
+ "Resource" => [ sqs_queue.arn ]
30
+ ]})
31
+ user.policies['info'] = info_policy
32
+ end
33
+
34
+ protected
35
+
36
+ def info_policy
37
+ JSON.pretty_generate({
38
+ "Statement" => [
39
+ "Effect" => "Allow",
40
+ "Action" => [ "sqs:ListQueues", "sqs:GetQueueUrl" ],
41
+ "Resource" => [ "*" ]
42
+ ]
43
+ })
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,187 @@
1
+ require 'thread'
2
+
3
+ # Shamelessly stolen from puma ;-)
4
+
5
+ # A simple thread pool management object.
6
+ #
7
+ module SQS::Job
8
+ class ThreadPool
9
+
10
+ # Maintain a minimum of +min+ and maximum of +max+ threads
11
+ # in the pool.
12
+ #
13
+ # The block passed is the work that will be performed in each
14
+ # thread.
15
+ #
16
+ def initialize(min, max, *extra, &block)
17
+ @cond = ConditionVariable.new
18
+ @mutex = Mutex.new
19
+
20
+ @todo = []
21
+
22
+ @spawned = 0
23
+ @waiting = 0
24
+
25
+ @min = Integer(min)
26
+ @max = Integer(max)
27
+ @block = block
28
+ @extra = extra
29
+
30
+ @shutdown = false
31
+
32
+ @trim_requested = 0
33
+
34
+ @workers = []
35
+
36
+ @auto_trim = nil
37
+
38
+ @mutex.synchronize do
39
+ @min.times { spawn_thread }
40
+ end
41
+ end
42
+
43
+ attr_reader :spawned, :trim_requested
44
+
45
+ # How many objects have yet to be processed by the pool?
46
+ #
47
+ def backlog
48
+ @mutex.synchronize { @todo.size }
49
+ end
50
+
51
+ # :nodoc:
52
+ #
53
+ # Must be called with @mutex held!
54
+ #
55
+ def spawn_thread
56
+ @spawned += 1
57
+
58
+ th = Thread.new do
59
+ todo = @todo
60
+ block = @block
61
+ mutex = @mutex
62
+ cond = @cond
63
+
64
+ extra = @extra.map { |i| i.new }
65
+
66
+ while true
67
+ work = nil
68
+
69
+ continue = true
70
+
71
+ mutex.synchronize do
72
+ while todo.empty?
73
+ if @trim_requested > 0
74
+ @trim_requested -= 1
75
+ continue = false
76
+ break
77
+ end
78
+
79
+ if @shutdown
80
+ continue = false
81
+ break
82
+ end
83
+
84
+ @waiting += 1
85
+ cond.wait mutex
86
+ @waiting -= 1
87
+ end
88
+
89
+ work = todo.pop if continue
90
+ end
91
+
92
+ break unless continue
93
+
94
+ block.call(work, *extra)
95
+ end
96
+
97
+ mutex.synchronize do
98
+ @spawned -= 1
99
+ @workers.delete th
100
+ end
101
+ end
102
+
103
+ @workers << th
104
+
105
+ th
106
+ end
107
+
108
+ private :spawn_thread
109
+
110
+ # Add +work+ to the todo list for a Thread to pickup and process.
111
+ def <<(work)
112
+ @mutex.synchronize do
113
+ if @shutdown
114
+ raise "Unable to add work while shutting down"
115
+ end
116
+
117
+ @todo << work
118
+
119
+ if @waiting == 0 and @spawned < @max
120
+ spawn_thread
121
+ end
122
+
123
+ @cond.signal
124
+ end
125
+ end
126
+
127
+ # If too many threads are in the pool, tell one to finish go ahead
128
+ # and exit. If +force+ is true, then a trim request is requested
129
+ # even if all threads are being utilized.
130
+ #
131
+ def trim(force=false)
132
+ @mutex.synchronize do
133
+ if (force or @waiting > 0) and @spawned - @trim_requested > @min
134
+ @trim_requested += 1
135
+ @cond.signal
136
+ end
137
+ end
138
+ end
139
+
140
+ class AutoTrim
141
+ def initialize(pool, timeout)
142
+ @pool = pool
143
+ @timeout = timeout
144
+ @running = false
145
+ end
146
+
147
+ def start!
148
+ @running = true
149
+
150
+ @thread = Thread.new do
151
+ while @running
152
+ @pool.trim
153
+ sleep @timeout
154
+ end
155
+ end
156
+ end
157
+
158
+ def stop
159
+ @running = false
160
+ @thread.wakeup
161
+ end
162
+ end
163
+
164
+ def auto_trim!(timeout=5)
165
+ @auto_trim = AutoTrim.new(self, timeout)
166
+ @auto_trim.start!
167
+ end
168
+
169
+ # Tell all threads in the pool to exit and wait for them to finish.
170
+ #
171
+ def shutdown
172
+ @mutex.synchronize do
173
+ @shutdown = true
174
+ @cond.broadcast
175
+
176
+ @auto_trim.stop if @auto_trim
177
+ end
178
+
179
+ # Use this instead of #each so that we don't stop in the middle
180
+ # of each and see a mutated object mid #each
181
+ @workers.first.join until @workers.empty?
182
+
183
+ @spawned = 0
184
+ @workers = []
185
+ end
186
+ end
187
+ end
@@ -0,0 +1,5 @@
1
+ module SQS
2
+ module Job
3
+ VERSION = "1.0.0"
4
+ end
5
+ end
@@ -0,0 +1,39 @@
1
+ module SQS::Job
2
+ # A Worker maintains a SQS::Job::ThreadPool and AWS::SQS::Queue
3
+ # and creates SQS::Job::Handler instances to process each message
4
+ # received. It is also responsible for boot/configuration
5
+ # stuff. There should only be one worker per process.
6
+ class Worker
7
+ attr_reader :queue
8
+
9
+ def initialize(queue)
10
+ @queue = queue
11
+ end
12
+
13
+ def run options = {}
14
+ require 'sqs/job/thread_pool'
15
+
16
+ min_threads = options[:min_threads] || SQS::Job.min_threads
17
+ max_threads = options[:max_threads] || SQS::Job.max_threads
18
+ @pool = SQS::Job::ThreadPool.new min_threads, max_threads do |msg|
19
+ log_exceptions{ Handler.new(msg).run! }
20
+ end
21
+
22
+ while true
23
+ # KEG: it's not clear if queue.poll accepts message_attribute_names
24
+ queue.receive_messages(wait_time_seconds: 10, batch_size: 10, message_attribute_names: [ 'signature', 'key_fingerprint' ]) do |msg|
25
+ @pool << msg
26
+ end
27
+ end
28
+ end
29
+
30
+ def log_exceptions &block
31
+ begin
32
+ block.call
33
+ rescue => ex
34
+ SQS::Job.logger.error "Error processing message: #{ex.class.name} #{ex}\n\t#{ex.backtrace.join("\t\n")}"
35
+ raise ex unless ex.is_a?(UnrecoverableException)
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,92 @@
1
+ require 'spec_helper'
2
+ require 'base64'
3
+
4
+ describe SQS::Job::Handler do
5
+ let(:message_type){ 'dummy' }
6
+ let(:message_params){ {'foo' => 'bar'} }
7
+ let(:message_hash){ { type: message_type, params: message_params } }
8
+ let(:message_body){ JSON.generate(message_hash) }
9
+ let(:key) { KEY }
10
+ let(:message_attributes) {
11
+ {
12
+ 'key_fingerprint' => {
13
+ string_value: key.fingerprint
14
+ },
15
+ 'signature' => {
16
+ string_value: Base64.strict_encode64(key.sign(message_body))
17
+ }
18
+ }
19
+ }
20
+ let(:sqs_message){ double('AWS::SQS::Message', body: message_body, message_attributes: message_attributes) }
21
+ let(:message_instance){ double('message instance', :"invoke!" => 'invoked', :"valid?" => true) }
22
+ let(:message_class){ double('message class', new: message_instance) }
23
+ let(:handler){ SQS::Job::Handler.new sqs_message }
24
+ subject { handler }
25
+
26
+ before do
27
+ allow(handler).to receive(:require).and_call_original
28
+ allow(handler).to receive(:require).with("sqs/job/message/dummy")
29
+ stub_const("SQS::Job::Message::Dummy", message_class)
30
+ allow(SQS::Job).to receive(:signing_keys).and_return [ key ]
31
+ end
32
+
33
+ describe "#run!" do
34
+ context "when message does not contain 'type'" do
35
+ let(:message_hash){ {} }
36
+ it "fails the message permanently" do
37
+ expect{ handler.run! }.to raise_error(MissingTypeException)
38
+ end
39
+ end
40
+
41
+ context "when type is 'base'" do
42
+ let(:message_type){ 'base' }
43
+ it "fails the message permanently" do
44
+ expect{ handler.run! }.to raise_error(UnrecognizedMessageTypeException, 'base')
45
+ end
46
+ end
47
+
48
+ context "when type is present" do
49
+ context "with invalid message signature" do
50
+ before do
51
+ expect(SQS::Job).to receive(:signing_keys).and_return [ ]
52
+ end
53
+ it "fails the message permanently" do
54
+ expect{ subject.run! }.to raise_error(SignatureInvalidException)
55
+ end
56
+ end
57
+
58
+ context "with an unknown type" do
59
+ let(:message_type){ 'foobar' }
60
+ it "fails the message permanently" do
61
+ expect{ handler.run! }.to raise_error(UnrecognizedMessageTypeException, 'foobar')
62
+ end
63
+ end
64
+
65
+ context "with an invalid message" do
66
+ it "fails the message permanently" do
67
+ allow(message_instance).to receive(:valid?).and_return false
68
+ expect(message_instance).to receive(:errors).and_return double('errors', full_messages: ['full-messages'])
69
+ expect(message_instance).not_to receive(:invoke!)
70
+ expect{ subject.run! }.to raise_error(InvalidMessageException, "full-messages")
71
+ end
72
+ end
73
+
74
+ context "with a valid message" do
75
+ it "requires 'sqs/job/message/dummy'" do
76
+ expect(handler).to receive(:require).with('sqs/job/message/dummy').and_return true
77
+ subject.run!
78
+ end
79
+
80
+ it "creates a message instance message body's params" do
81
+ expect(message_class).to receive(:new).with(message_params)
82
+ subject.run!
83
+ end
84
+
85
+ it "calls invoke! on the message instance" do
86
+ expect(message_instance).to receive(:invoke!)
87
+ subject.run!
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,61 @@
1
+ require 'simplecov'
2
+ SimpleCov.start do
3
+ add_filter "/spec/"
4
+ end
5
+
6
+ require 'rubygems'
7
+ $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
8
+ $:.unshift File.join(File.dirname(__FILE__), "lib")
9
+
10
+ # Allows loading of an environment config based on the environment
11
+ require 'rspec'
12
+
13
+ # Uncomment the next line to use webrat's matchers
14
+ #require 'webrat/integrations/rspec-rails'
15
+
16
+ RSpec.configure do |config|
17
+ # If you're not using ActiveRecord you should remove these
18
+ # lines, delete config/database.yml and disable :active_record
19
+ # in your config/boot.rb
20
+ #config.use_transactional_fixtures = true
21
+ #config.use_instantiated_fixtures = false
22
+ #config.fixture_path = File.join(redmine_root, 'test', 'fixtures')
23
+
24
+ # == Fixtures
25
+ #
26
+ # You can declare fixtures for each example_group like this:
27
+ # describe "...." do
28
+ # fixtures :table_a, :table_b
29
+ #
30
+ # Alternatively, if you prefer to declare them only once, you can
31
+ # do so right here. Just uncomment the next line and replace the fixture
32
+ # names with your fixtures.
33
+ #
34
+ #
35
+ # If you declare global fixtures, be aware that they will be declared
36
+ # for all of your examples, even those that don't use them.
37
+ #
38
+ # You can also declare which fixtures to use (for example fixtures for test/fixtures):
39
+ #
40
+ # config.fixture_path = RAILS_ROOT + '/spec/fixtures/'
41
+ #
42
+ # == Mock Framework
43
+ #
44
+ # RSpec uses its own mocking framework by default. If you prefer to
45
+ # use mocha, flexmock or RR, uncomment the appropriate line:
46
+ #
47
+ # config.mock_with :mocha
48
+ # config.mock_with :flexmock
49
+ # config.mock_with :rr
50
+ #
51
+ # == Notes
52
+ #
53
+ # For more information take a look at Spec::Runner::Configuration and Spec::Runner
54
+ end
55
+
56
+ Dir[File.expand_path(File.join(File.dirname(__FILE__),'support','**','*.rb'))].each {|f| require f}
57
+
58
+ require 'sqs/job'
59
+ require 'slosilo'
60
+
61
+ KEY = Slosilo::Key.new
@@ -0,0 +1,34 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sqs/job/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "sqs-job"
8
+ spec.version = SQS::Job::VERSION
9
+ spec.authors = ["Jon Mason", "Kevin Gilpin"]
10
+ spec.email = ["jonathan.j.mason@gmail.com", "kgilpin@gmail.com"]
11
+ spec.summary = %q{Simple job processing library which uses SQS.}
12
+ spec.homepage = "https://github.com/conjurinc/sqs-job"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_dependency "validatable"
21
+ spec.add_dependency "activesupport"
22
+ spec.add_dependency "aws-sdk"
23
+ spec.add_dependency "slosilo"
24
+
25
+ spec.add_development_dependency "bundler", "~> 1.6"
26
+ spec.add_development_dependency "conjur-api"
27
+ spec.add_development_dependency "conjur-cli"
28
+ spec.add_development_dependency "rake"
29
+ spec.add_development_dependency "rspec"
30
+ spec.add_development_dependency "cucumber"
31
+ spec.add_development_dependency "ci_reporter_rspec"
32
+ spec.add_development_dependency 'ci_reporter_cucumber'
33
+ spec.add_development_dependency "simplecov"
34
+ end
metadata ADDED
@@ -0,0 +1,257 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sqs-job
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Jon Mason
8
+ - Kevin Gilpin
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2014-10-20 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: validatable
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - '>='
19
+ - !ruby/object:Gem::Version
20
+ version: '0'
21
+ type: :runtime
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - '>='
26
+ - !ruby/object:Gem::Version
27
+ version: '0'
28
+ - !ruby/object:Gem::Dependency
29
+ name: activesupport
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - '>='
33
+ - !ruby/object:Gem::Version
34
+ version: '0'
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - '>='
40
+ - !ruby/object:Gem::Version
41
+ version: '0'
42
+ - !ruby/object:Gem::Dependency
43
+ name: aws-sdk
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - '>='
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ type: :runtime
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - '>='
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ - !ruby/object:Gem::Dependency
57
+ name: slosilo
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - '>='
61
+ - !ruby/object:Gem::Version
62
+ version: '0'
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: bundler
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ~>
75
+ - !ruby/object:Gem::Version
76
+ version: '1.6'
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ~>
82
+ - !ruby/object:Gem::Version
83
+ version: '1.6'
84
+ - !ruby/object:Gem::Dependency
85
+ name: conjur-api
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ type: :development
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - '>='
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ - !ruby/object:Gem::Dependency
99
+ name: conjur-cli
100
+ requirement: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - '>='
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ type: :development
106
+ prerelease: false
107
+ version_requirements: !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - '>='
110
+ - !ruby/object:Gem::Version
111
+ version: '0'
112
+ - !ruby/object:Gem::Dependency
113
+ name: rake
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - '>='
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ type: :development
120
+ prerelease: false
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ - !ruby/object:Gem::Dependency
127
+ name: rspec
128
+ requirement: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - '>='
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ type: :development
134
+ prerelease: false
135
+ version_requirements: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - '>='
138
+ - !ruby/object:Gem::Version
139
+ version: '0'
140
+ - !ruby/object:Gem::Dependency
141
+ name: cucumber
142
+ requirement: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - '>='
145
+ - !ruby/object:Gem::Version
146
+ version: '0'
147
+ type: :development
148
+ prerelease: false
149
+ version_requirements: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - '>='
152
+ - !ruby/object:Gem::Version
153
+ version: '0'
154
+ - !ruby/object:Gem::Dependency
155
+ name: ci_reporter_rspec
156
+ requirement: !ruby/object:Gem::Requirement
157
+ requirements:
158
+ - - '>='
159
+ - !ruby/object:Gem::Version
160
+ version: '0'
161
+ type: :development
162
+ prerelease: false
163
+ version_requirements: !ruby/object:Gem::Requirement
164
+ requirements:
165
+ - - '>='
166
+ - !ruby/object:Gem::Version
167
+ version: '0'
168
+ - !ruby/object:Gem::Dependency
169
+ name: ci_reporter_cucumber
170
+ requirement: !ruby/object:Gem::Requirement
171
+ requirements:
172
+ - - '>='
173
+ - !ruby/object:Gem::Version
174
+ version: '0'
175
+ type: :development
176
+ prerelease: false
177
+ version_requirements: !ruby/object:Gem::Requirement
178
+ requirements:
179
+ - - '>='
180
+ - !ruby/object:Gem::Version
181
+ version: '0'
182
+ - !ruby/object:Gem::Dependency
183
+ name: simplecov
184
+ requirement: !ruby/object:Gem::Requirement
185
+ requirements:
186
+ - - '>='
187
+ - !ruby/object:Gem::Version
188
+ version: '0'
189
+ type: :development
190
+ prerelease: false
191
+ version_requirements: !ruby/object:Gem::Requirement
192
+ requirements:
193
+ - - '>='
194
+ - !ruby/object:Gem::Version
195
+ version: '0'
196
+ description:
197
+ email:
198
+ - jonathan.j.mason@gmail.com
199
+ - kgilpin@gmail.com
200
+ executables: []
201
+ extensions: []
202
+ extra_rdoc_files: []
203
+ files:
204
+ - .gitignore
205
+ - .project
206
+ - Gemfile
207
+ - LICENSE.txt
208
+ - README.md
209
+ - Rakefile
210
+ - cucumber-policy.rb
211
+ - features/send_message.feature
212
+ - features/sqs/job/message/test_message.rb
213
+ - features/step_definitions/job_steps.rb
214
+ - features/support/env.rb
215
+ - lib/sqs/job.rb
216
+ - lib/sqs/job/exceptions.rb
217
+ - lib/sqs/job/handler.rb
218
+ - lib/sqs/job/message/base.rb
219
+ - lib/sqs/job/policy.rb
220
+ - lib/sqs/job/provisioner.rb
221
+ - lib/sqs/job/thread_pool.rb
222
+ - lib/sqs/job/version.rb
223
+ - lib/sqs/job/worker.rb
224
+ - spec/handler_spec.rb
225
+ - spec/spec_helper.rb
226
+ - sqs-job.gemspec
227
+ homepage: https://github.com/conjurinc/sqs-job
228
+ licenses:
229
+ - MIT
230
+ metadata: {}
231
+ post_install_message:
232
+ rdoc_options: []
233
+ require_paths:
234
+ - lib
235
+ required_ruby_version: !ruby/object:Gem::Requirement
236
+ requirements:
237
+ - - '>='
238
+ - !ruby/object:Gem::Version
239
+ version: '0'
240
+ required_rubygems_version: !ruby/object:Gem::Requirement
241
+ requirements:
242
+ - - '>='
243
+ - !ruby/object:Gem::Version
244
+ version: '0'
245
+ requirements: []
246
+ rubyforge_project:
247
+ rubygems_version: 2.2.2
248
+ signing_key:
249
+ specification_version: 4
250
+ summary: Simple job processing library which uses SQS.
251
+ test_files:
252
+ - features/send_message.feature
253
+ - features/sqs/job/message/test_message.rb
254
+ - features/step_definitions/job_steps.rb
255
+ - features/support/env.rb
256
+ - spec/handler_spec.rb
257
+ - spec/spec_helper.rb