sqewer 6.2.2 → 6.5.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 70857d657ec0e0190310f806f8b34a8c04df7504c67e9890cd2ee2580e5ef387
4
- data.tar.gz: e3d7d1fa311e25ac8e53b58db842446cff851924cba36583bc9bf87a9a5120b9
3
+ metadata.gz: cf1c2d6717fda50ce8a72721d00ffcfd2fae867f40f40a07a0cf48cbe94fe17b
4
+ data.tar.gz: fb1622a03a4f540aaa96d24dc3d5d39c5d400b658f371c88dc682844ac784ea5
5
5
  SHA512:
6
- metadata.gz: ba846399144de508eff227a61e7d78e6fa662884565670183f9992530e06790b704c013b866436fec0f9f860872908a528db6db8751a711811480b9f9f7b0e5f
7
- data.tar.gz: 708a29a0e29253143002c639a963dc65e966641045113c4a826a84536935969f5906da291d865042e3655da5399eb5d9e95805d96d306462bf5dd2266f924dd7
6
+ metadata.gz: fd4e73838959531d646d0ed69fd8316e85075d3abe96b5362c58ff5de9af27b26cec142273bce674aaadf571275f1f91cf055c305be4b9797edd75eb89b74222
7
+ data.tar.gz: 9f4b9006f504bd4a0a85abea9d3c82bf2c70ccdfecf59095df58a1e6b1b8b3ed0b6ba819778f828c48e5dd1865c1f312c5feda78485e9ceab95565e9559cc870
data/.gitignore CHANGED
@@ -1,54 +1,56 @@
1
- .rspec
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+
13
+ # Used by dotenv library to load environment variables.
2
14
  .env
3
15
 
4
- # rcov generated
5
- coverage
6
- coverage.data
16
+ # Ignore Byebug command history file.
17
+ .byebug_history
7
18
 
8
- # rdoc generated
9
- rdoc
19
+ ## Specific to RubyMotion:
20
+ .dat*
21
+ .repl_history
22
+ build/
23
+ *.bridgesupport
24
+ build-iPhoneOS/
25
+ build-iPhoneSimulator/
10
26
 
11
- # yard generated
12
- doc
13
- .yardoc
14
-
15
- # bundler
16
- .bundle
17
- Gemfile.lock
18
- gemfiles/*.lock
19
-
20
- # jeweler generated
21
- pkg
22
-
23
- # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore:
27
+ ## Specific to RubyMotion (use of CocoaPods):
24
28
  #
25
- # * Create a file at ~/.gitignore
26
- # * Include files you want ignored
27
- # * Run: git config --global core.excludesfile ~/.gitignore
29
+ # We recommend against adding the Pods directory to your .gitignore. However
30
+ # you should judge for yourself, the pros and cons are mentioned at:
31
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
28
32
  #
29
- # After doing this, these files will be ignored in all your git projects,
30
- # saving you from having to 'pollute' every project you touch with them
31
- #
32
- # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
33
- #
34
- # For MacOS:
35
- #
36
- #.DS_Store
33
+ # vendor/Pods/
37
34
 
38
- # For TextMate
39
- #*.tmproj
40
- #tmtags
35
+ ## Documentation cache and generated files:
36
+ /.yardoc/
37
+ /_yardoc/
38
+ /doc/
39
+ /rdoc/
41
40
 
42
- # For emacs:
43
- #*~
44
- #\#*
45
- #.\#*
41
+ ## Environment normalization:
42
+ /.bundle/
43
+ /vendor/bundle
44
+ /lib/bundler/man/
46
45
 
47
- # For vim:
48
- #*.swp
46
+ # for a library or gem, you might want to ignore these files since the code is
47
+ # intended to run in multiple environments; otherwise, check them in:
48
+ Gemfile.lock
49
+ .ruby-version
50
+ .ruby-gemset
49
51
 
50
- # For redcar:
51
- #.redcar
52
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
53
+ .rvmrc
52
54
 
53
- # For rubinius:
54
- #*.rbc
55
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
56
+ # .rubocop-https?--*
data/.travis.yml CHANGED
@@ -6,6 +6,8 @@ rvm:
6
6
  - 2.3.7
7
7
  - 2.4.5
8
8
  - 2.5.1
9
+ - 2.6.5
10
+ - 2.7.0
9
11
  cache: bundler
10
12
  sudo: false
11
13
  env:
@@ -19,3 +21,7 @@ matrix:
19
21
  gemfile: gemfiles/Gemfile.rails-5.1.x
20
22
  - rvm: 2.5.1
21
23
  gemfile: gemfiles/Gemfile.rails-4.2.x
24
+ - rvm: 2.6.5
25
+ gemfile: gemfiles/Gemfile.rails-4.2.x
26
+ - rvm: 2.7.0
27
+ gemfile: gemfiles/Gemfile.rails-4.2.x
data/CHANGELOG.md CHANGED
@@ -1,3 +1,23 @@
1
+ ### 6.5.1
2
+ - Also retry on `Aws::SQS::Errors::InternalError` exception when receiving/sending messages. This will make
3
+ the receiving thread more resilient to sudden SQS failures. By the time SQS recovers the receiving thread
4
+ should stay alive.
5
+
6
+ ### 6.5.0
7
+ - Adds `$stdout.sync = true` to CLI to flush the logs to STDOUT
8
+
9
+ ### 6.4.1
10
+ - Retrieve attributes when receiving messages from SQS
11
+
12
+ ### 6.4.0
13
+ - Raise an exception in submit! if the job serializes to a message that is
14
+ above the native SQS limit for message size.
15
+ - Ensure SendMessageBatch is only performed for batches totaling 256KB of message size or less.
16
+ - Insert Sqewer::Error between StandardError and our custom errors for easier rescuing
17
+
18
+ ### 6.3.0
19
+ - Add support for Ruby 2.7
20
+
1
21
  ### 6.2.2
2
22
  - Test the Appsignal integration using actual Appsignal libraries
3
23
  - In the Appsignal integration, replace a call of `set_queue_start=` with `set_queue_start`
data/README.md CHANGED
@@ -20,14 +20,14 @@ and to start processing, in your commandline handler:
20
20
 
21
21
  #!/usr/bin/env ruby
22
22
  require 'my_applicaion'
23
- Sqewer::CLI.run
23
+ Sqewer::CLI.start
24
24
 
25
25
  To add arguments to the job
26
26
 
27
27
  class JobWithArgs
28
28
  include Sqewer::SimpleJob
29
29
  attr_accessor :times
30
-
30
+
31
31
  def run
32
32
  ...
33
33
  end
@@ -88,6 +88,10 @@ Note that at this point only arguments that are raw JSON types are supported:
88
88
 
89
89
  If you need marshalable Ruby types there instead, you might need to implement a custom `Serializer.`
90
90
 
91
+ ### Sqewer::SimpleJob
92
+
93
+ The module `Sqewer::SimpleJob` can be included to a job class to add some features, specially dealing with attributes, see more details [here](https://github.com/WeTransfer/sqewer/blob/master/lib/sqewer/simple_job.rb).
94
+
91
95
  ## Jobs spawning dependent jobs
92
96
 
93
97
  If your `run` method on the job object accepts arguments (has non-zero `arity` ) the `ExecutionContext` will
@@ -119,11 +123,11 @@ include all the keyword arguments needed to instantiate the job when executing.
119
123
  def initialize(to:, body:)
120
124
  ...
121
125
  end
122
-
126
+
123
127
  def run()
124
128
  ...
125
129
  end
126
-
130
+
127
131
  def to_h
128
132
  {to: @to, body: @body}
129
133
  end
@@ -151,7 +155,7 @@ conform to the job serialization format used internally. For example, you can ha
151
155
  message = JSON.load(message_blob)
152
156
  return if message['Service'] # AWS test
153
157
  return HandleS3Notification.new(message) if message['Records']
154
-
158
+
155
159
  super # as default
156
160
  end
157
161
  end
@@ -176,7 +180,7 @@ The very minimal executable for running jobs would be this:
176
180
 
177
181
  #!/usr/bin/env ruby
178
182
  require 'my_applicaion'
179
- Sqewer::CLI.run
183
+ Sqewer::CLI.start
180
184
 
181
185
  This will connect to the queue at the URL set in the `SQS_QUEUE_URL` environment variable, and
182
186
  use all the default parameters. The `CLI` module will also set up a signal handler to terminate
@@ -233,9 +237,9 @@ you generate. For example, you could use a pipe. But in a more general case some
233
237
  ActiveRAMGobbler.fetch_stupendously_many_things.each do |...|
234
238
  end
235
239
  end
236
-
240
+
237
241
  _, status = Process.wait2(pid)
238
-
242
+
239
243
  # Raise an error in the parent process to signal Sqewer that the job failed
240
244
  # if the child exited with a non-0 status
241
245
  raise "Child process crashed" unless status.exitstatus && status.exitstatus.zero?
@@ -252,7 +256,7 @@ You can wrap job processing in middleware. A full-featured middleware class look
252
256
  # msg_id is the receipt handle, msg_payload is the message body string, msg_attributes are the message's attributes
253
257
  yield
254
258
  end
255
-
259
+
256
260
  # Surrounds the actual job execution
257
261
  def around_execution(job, context)
258
262
  # job is the actual job you will be running, context is the ExecutionContext.
@@ -378,7 +382,7 @@ products using this library, was a very bad idea (more workload for deployment).
378
382
 
379
383
  ## Why so many configurable components?
380
384
 
381
- Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your
385
+ Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your
382
386
  implementation in instead. One product needs foreign-submitted SQS jobs (S3 notifications). Another product
383
387
  needs a custom Logger subclass. Yet another product needs process-based concurrency on top of threads.
384
388
  Yet another process needs to manage database connections when running the jobs. Have 3-4 of those, and a
@@ -398,7 +402,7 @@ Because I found that a producer-consumer model with a thread pool works quite we
398
402
  the Ruby standard library alone.
399
403
 
400
404
  ## Contributing to the library
401
-
405
+
402
406
  * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
403
407
  * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
404
408
  * Fork the project.
@@ -4,5 +4,5 @@ source "http://rubygems.org"
4
4
  gemspec path: __dir__ + '/..'
5
5
 
6
6
  gem 'sqlite3', '~> 1.3.6'
7
- gem 'activejob', "~> 4.2"
8
- gem 'activerecord', "~> 4.2"
7
+ gem 'activejob', "~> 4.2.0"
8
+ gem 'activerecord', "~> 4.2.0"
@@ -4,5 +4,5 @@ source "http://rubygems.org"
4
4
  gemspec path: __dir__ + '/..'
5
5
 
6
6
  gem 'sqlite3', '~> 1.3.6'
7
- gem 'activejob', "~> 5", "< 5.1"
8
- gem 'activerecord', "~> 5", "< 5.1"
7
+ gem 'activejob', "~> 5.0.0"
8
+ gem 'activerecord', "~> 5.0.0"
@@ -3,6 +3,6 @@ source "http://rubygems.org"
3
3
  # Gemspec as base dependency set
4
4
  gemspec path: __dir__ + '/..'
5
5
 
6
- gem 'sqlite3', '~> 1.3.6'
7
- gem 'activejob', "~> 5.1", "< 5.2"
8
- gem 'activerecord', "~> 5.1", "< 5.2"
6
+ gem 'sqlite3', "~> 1.3", ">= 1.3.6"
7
+ gem 'activejob', "~> 5.1.0"
8
+ gem 'activerecord', "~> 5.1.0"
data/lib/sqewer.rb CHANGED
@@ -1,5 +1,8 @@
1
1
  # The enclosing module for the library
2
2
  module Sqewer
3
+ class Error < StandardError
4
+ end
5
+
3
6
  # Eager-load everything except extensions. Sort to ensure
4
7
  # the files load in the same order on all platforms.
5
8
  Dir.glob(__dir__ + '/**/*.rb').sort.each do |path|
@@ -23,7 +26,7 @@ module Sqewer
23
26
  def self.submit!(*jobs, **options)
24
27
  Sqewer::Submitter.default.submit!(*jobs, **options)
25
28
  end
26
-
29
+
27
30
  # If we are within Rails, load the railtie
28
31
  require_relative 'sqewer/extensions/railtie' if defined?(Rails)
29
32
 
data/lib/sqewer/cli.rb CHANGED
@@ -1,3 +1,4 @@
1
+ $stdout.sync = true
1
2
  # Wraps a Worker object in a process-wide commanline handler. Once the `start` method is
2
3
  # called, signal handlers will be installed for the following signals:
3
4
  #
@@ -11,7 +11,7 @@ class Sqewer::Connection
11
11
  MAX_RANDOM_FAILURES_PER_CALL = 10
12
12
  MAX_RANDOM_RECEIVE_FAILURES = 100 # sure to hit the max_elapsed_time of 900 seconds
13
13
 
14
- NotOurFaultAwsError = Class.new(StandardError)
14
+ NotOurFaultAwsError = Class.new(Sqewer::Error)
15
15
 
16
16
  # A wrapper for most important properties of the received message
17
17
  class Message < Struct.new(:receipt_handle, :body, :attributes)
@@ -52,11 +52,15 @@ class Sqewer::Connection
52
52
  # even after 15 minutes it is either down or the server is misconfigured. Either way it makes no sense to
53
53
  # continue.
54
54
  #
55
- # @return [Array<Message>] an array of Message objects
55
+ # @return [Array<Message>] an array of Message objects
56
56
  def receive_messages
57
- Retriable.retriable on: Seahorse::Client::NetworkingError, tries: MAX_RANDOM_RECEIVE_FAILURES do
58
- response = client.receive_message(queue_url: @queue_url,
59
- wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: BATCH_RECEIVE_SIZE)
57
+ Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_RECEIVE_FAILURES do
58
+ response = client.receive_message(
59
+ queue_url: @queue_url,
60
+ attribute_names: ['All'],
61
+ wait_time_seconds: DEFAULT_TIMEOUT_SECONDS,
62
+ max_number_of_messages: BATCH_RECEIVE_SIZE
63
+ )
60
64
  response.messages.map {|message| Message.new(message.receipt_handle, message.body, message.attributes) }
61
65
  end
62
66
  end
@@ -65,7 +69,7 @@ class Sqewer::Connection
65
69
  #
66
70
  # @param message_body[String] the message to send
67
71
  # @param kwargs_for_send[Hash] additional arguments for the submit (such as `delay_seconds`).
68
- # Passes the arguments to the AWS SDK.
72
+ # Passes the arguments to the AWS SDK.
69
73
  # @return [void]
70
74
  def send_message(message_body, **kwargs_for_send)
71
75
  send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
@@ -91,6 +95,58 @@ class Sqewer::Connection
91
95
  m[:delay_seconds] = kwargs_for_send[:delay_seconds] if kwargs_for_send[:delay_seconds]
92
96
  messages << m
93
97
  end
98
+
99
+ # each_batch here also needs to ensure that the sum of payload lengths does not exceed 256kb
100
+ def each_batch
101
+ regrouped = pack_into_batches(messages, weight_limit: 256 * 1024, batch_length_limit: 10) do |message|
102
+ message.fetch(:message_body).bytesize
103
+ end
104
+ regrouped.each { |b| yield(b) }
105
+ end
106
+
107
+ # Optimizes a large list of items into batches of 10 items
108
+ # or less and with the sum of item lengths being below 256KB
109
+ # The block given to the method should return the weight of the given item
110
+ def pack_into_batches(items, weight_limit:, batch_length_limit:)
111
+ batches = []
112
+ current_batch = []
113
+ current_batch_weight = 0
114
+
115
+ # Sort the items by their weight (length of the body).
116
+ sorted_items = items.sort_by { |item| yield(item) }
117
+
118
+ # and then take 1 item from the list and append it to the batch if it fits.
119
+ # If it doesn't fit, no item after it will fit into this batch either (!)
120
+ # which is how we can optimize
121
+ sorted_items.each_with_index do |item|
122
+ weight_of_this_item = yield(item)
123
+
124
+ # First protect from invalid input
125
+ if weight_of_this_item > weight_limit
126
+ raise "#{item.inspect} was larger than the permissible limit"
127
+ # The first limit is on the item count per batch -
128
+ # if we are limited on that the batch needs to be closed
129
+ elsif current_batch.length == batch_length_limit
130
+ batches << current_batch
131
+ current_batch = []
132
+ current_batch_weight = 0
133
+ # If placing this item in the batch would make the batch overweight
134
+ # we need to close the batch, because all the items which come after
135
+ # this one will be same size or larger. This is the key part of the optimization.
136
+ elsif (current_batch_weight + weight_of_this_item) > weight_limit
137
+ batches << current_batch
138
+ current_batch = []
139
+ current_batch_weight = 0
140
+ end
141
+
142
+ # and then append the item to the current batch
143
+ current_batch_weight += weight_of_this_item
144
+ current_batch << item
145
+ end
146
+ batches << current_batch unless current_batch.empty?
147
+
148
+ batches
149
+ end
94
150
  end
95
151
 
96
152
  # Saves the receipt handles to batch-delete from the SQS queue
@@ -135,8 +191,12 @@ class Sqewer::Connection
135
191
 
136
192
  private
137
193
 
194
+ def network_and_aws_sdk_errors
195
+ [NotOurFaultAwsError, Seahorse::Client::NetworkingError, Aws::SQS::Errors::InternalError]
196
+ end
197
+
138
198
  def handle_batch_with_retries(method, batch)
139
- Retriable.retriable on: [NotOurFaultAwsError, Seahorse::Client::NetworkingError], tries: MAX_RANDOM_FAILURES_PER_CALL do
199
+ Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_FAILURES_PER_CALL do
140
200
  resp = client.send(method, queue_url: @queue_url, entries: batch)
141
201
  wrong_messages, aws_failures = resp.failed.partition {|m| m.sender_fault }
142
202
  if wrong_messages.any?
@@ -4,7 +4,6 @@ module Sqewer
4
4
  # to Appsignal and to monitor performance. Will only activate
5
5
  # if the Appsignal gem is loaded within the current process and active.
6
6
  class AppsignalWrapper
7
-
8
7
  def self.new
9
8
  if defined?(Appsignal)
10
9
  super
@@ -13,7 +13,7 @@ class Sqewer::Serializer
13
13
  @instance ||= new
14
14
  end
15
15
 
16
- AnonymousJobClass = Class.new(StandardError)
16
+ AnonymousJobClass = Class.new(Sqewer::Error)
17
17
 
18
18
  # Instantiate a Job object from a message body string. If the
19
19
  # returned result is `nil`, the job will be skipped.
@@ -33,18 +33,18 @@ class Sqewer::Serializer
33
33
  # use a default that will put us ahead of that execution deadline from the start.
34
34
  t = Time.now.to_i
35
35
  execute_after = job_ticket_hash.fetch(:_execute_after) { t - 5 }
36
-
36
+
37
37
  job_params = job_ticket_hash.delete(:_job_params)
38
38
  job = if job_params.nil? || job_params.empty?
39
39
  job_class.new # no args
40
40
  else
41
41
  job_class.new(**job_params) # The rest of the message are keyword arguments for the job
42
42
  end
43
-
44
- # If the job is not up for execution now, wrap it with something that will
43
+
44
+ # If the job is not up for execution now, wrap it with something that will
45
45
  # re-submit it for later execution when the run() method is called
46
46
  return ::Sqewer::Resubmit.new(job, execute_after) if execute_after > t
47
-
47
+
48
48
  job
49
49
  end
50
50
 
@@ -65,7 +65,7 @@ class Sqewer::Serializer
65
65
  job_params = job.respond_to?(:to_h) ? job.to_h : nil
66
66
  job_ticket_hash = {_job_class: job_class_name, _job_params: job_params}
67
67
  job_ticket_hash[:_execute_after] = execute_after_timestamp.to_i if execute_after_timestamp
68
-
68
+
69
69
  JSON.dump(job_ticket_hash)
70
70
  end
71
71
  end
@@ -4,9 +4,16 @@
4
4
  # * initialize() will have keyword access to all accessors, and will ensure you have called each one of them
5
5
  # * to_h() will produce a symbolized Hash with all the properties defined using attr_accessor, and the job_class_name
6
6
  # * inspect() will provide a sensible default string representation for logging
7
+ #
8
+ # This module validates if the attributes defined in the job class are the same as
9
+ # those persisted in the queue. More details on `Sqewer::SimpleJob#initialize`.
10
+ # Because of this, it's required to create a new job class when adding or removing
11
+ # an attribute.
12
+ # This mechanism guarantees strong consistency. Without it, a new deployed job class
13
+ # could process old incompatible payloads.
7
14
  module Sqewer::SimpleJob
8
- UnknownJobAttribute = Class.new(StandardError)
9
- MissingAttribute = Class.new(StandardError)
15
+ UnknownJobAttribute = Class.new(Sqewer::Error)
16
+ MissingAttribute = Class.new(Sqewer::Error)
10
17
 
11
18
  EQ_END = /(\w+)(\=)$/
12
19
 
@@ -53,7 +60,7 @@ module Sqewer::SimpleJob
53
60
  accessor = "#{k}="
54
61
  touched_attributes << k
55
62
  unless respond_to?(accessor)
56
- raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}"
63
+ raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}"
57
64
  end
58
65
 
59
66
  send("#{k}=", v)
@@ -3,7 +3,10 @@
3
3
  # and the serializer (something that responds to `#serialize`) to
4
4
  # convert the job into the string that will be put in the queue.
5
5
  class Sqewer::Submitter < Struct.new(:connection, :serializer)
6
- NotSqewerJob = Class.new(StandardError)
6
+ MAX_PERMITTED_MESSAGE_SIZE_BYTES = 256 * 1024
7
+
8
+ NotSqewerJob = Class.new(Sqewer::Error)
9
+ MessageTooLarge = Class.new(Sqewer::Error)
7
10
 
8
11
  # Returns a default Submitter, configured with the default connection
9
12
  # and the default serializer.
@@ -12,7 +15,7 @@ class Sqewer::Submitter < Struct.new(:connection, :serializer)
12
15
  end
13
16
 
14
17
  def submit!(job, **kwargs_for_send)
15
- raise NotSqewerJob.new("Submitted object is not a valid job: #{job.inspect}") unless job.respond_to?(:run)
18
+ validate_job_responds_to_run!(job)
16
19
  message_body = if delay_by_seconds = kwargs_for_send[:delay_seconds]
17
20
  clamped_delay = clamp_delay(delay_by_seconds)
18
21
  kwargs_for_send[:delay_seconds] = clamped_delay
@@ -21,11 +24,26 @@ class Sqewer::Submitter < Struct.new(:connection, :serializer)
21
24
  else
22
25
  serializer.serialize(job)
23
26
  end
27
+ validate_message_for_size!(message_body, job)
28
+
24
29
  connection.send_message(message_body, **kwargs_for_send)
25
30
  end
26
-
31
+
27
32
  private
28
-
33
+
34
+ def validate_job_responds_to_run!(job)
35
+ return if job.respond_to?(:run)
36
+ error_message = "Submitted object is not a valid job (does not respond to #run): #{job.inspect}"
37
+ raise NotSqewerJob.new(error_message)
38
+ end
39
+
40
+ def validate_message_for_size!(message_body, job)
41
+ actual_bytesize = message_body.bytesize
42
+ return if actual_bytesize <= MAX_PERMITTED_MESSAGE_SIZE_BYTES
43
+ error_message = "Job #{job.inspect} serialized to a message which was too large (#{actual_bytesize} bytes)"
44
+ raise MessageTooLarge.new(error_message)
45
+ end
46
+
29
47
  def clamp_delay(delay)
30
48
  [1, 899, delay].sort[1]
31
49
  end
@@ -1,3 +1,3 @@
1
1
  module Sqewer
2
- VERSION = '6.2.2'
2
+ VERSION = '6.5.1'
3
3
  end
data/lib/sqewer/worker.rb CHANGED
@@ -48,7 +48,7 @@ class Sqewer::Worker
48
48
  #
49
49
  # @param connection[Sqewer::Connection] the object that handles polling and submitting
50
50
  # @param serializer[#serialize, #unserialize] the serializer/unserializer for the jobs
51
- # @param execution_context_class[Class] the class for the execution context (will be instantiated by
51
+ # @param execution_context_class[Class] the class for the execution context (will be instantiated by
52
52
  # the worker for each job execution)
53
53
  # @param submitter_class[Class] the class used for submitting jobs (will be instantiated by the worker for each job execution)
54
54
  # @param middleware_stack[Sqewer::MiddlewareStack] the middleware stack that is going to be used
@@ -91,7 +91,7 @@ class Sqewer::Worker
91
91
 
92
92
  consumers = (1..@num_threads).map do
93
93
  Thread.new do
94
- catch(:goodbye) { loop {take_and_execute} }
94
+ loop { take_and_execute }
95
95
  end
96
96
  end
97
97
 
@@ -212,58 +212,34 @@ class Sqewer::Worker
212
212
  submitter = submitter_class.new(box, serializer)
213
213
  context = execution_context_class.new(submitter, {'logger' => logger})
214
214
 
215
- t = Time.now
215
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
216
216
  middleware_stack.around_execution(job, context) do
217
217
  job.method(:run).arity.zero? ? job.run : job.run(context)
218
+ # delete_message will enqueue the message for deletion,
219
+ # but when flush! is called _first_ the messages pending will be
220
+ # delivered to the queue, THEN all the deletes are going to be performed.
221
+ # So it is safe to call delete here first - the delete won't get to the queue
222
+ # if flush! fails to spool pending messages.
223
+ box.delete_message(message.receipt_handle)
224
+ n_flushed = box.flush!
225
+ logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed > 0
218
226
  end
219
- box.delete_message(message.receipt_handle)
220
227
 
221
- delta = Time.now - t
228
+ delta = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
222
229
  logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
223
- ensure
224
- n_flushed = box.flush!
225
- logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
226
230
  end
227
231
 
228
232
  def take_and_execute
229
233
  message = @execution_queue.pop(nonblock=true)
230
234
  handle_message(message)
231
235
  rescue ThreadError # Queue is empty
232
- throw :goodbye if stopping?
233
- sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
236
+ if stopping?
237
+ Thread.current.exit
238
+ else
239
+ sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
240
+ end
234
241
  rescue => e # anything else, at or below StandardError that does not need us to quit
235
242
  @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..64], e.class, e.message] }
236
243
  e.backtrace.each { |s| @logger.debug{"\t#{s}"} }
237
244
  end
238
-
239
- def perform(message)
240
- # Create a messagebox that buffers all the calls to Connection, so that
241
- # we can send out those commands in one go (without interfering with senders
242
- # on other threads, as it seems the Aws::SQS::Client is not entirely
243
- # thread-safe - or at least not it's HTTP client part).
244
- box = Sqewer::ConnectionMessagebox.new(connection)
245
-
246
- job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body, message.attributes) do
247
- serializer.unserialize(message.body)
248
- end
249
- return unless job
250
-
251
- submitter = submitter_class.new(box, serializer)
252
- context = execution_context_class.new(submitter, {'logger' => logger})
253
-
254
- t = Time.now
255
- middleware_stack.around_execution(job, context) do
256
- job.method(:run).arity.zero? ? job.run : job.run(context)
257
- end
258
-
259
- # Perform two flushes, one for any possible jobs the job has spawned,
260
- # and one for the job delete afterwards
261
- box.delete_message(message.receipt_handle)
262
-
263
- delta = Time.now - t
264
- logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
265
- ensure
266
- n_flushed = box.flush!
267
- logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
268
- end
269
245
  end
data/sqewer.gemspec CHANGED
@@ -6,8 +6,8 @@ require "sqewer/version"
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = "sqewer"
8
8
  spec.version = Sqewer::VERSION
9
- spec.authors = ["Julik Tarkhanov"]
10
- spec.email = ["me@julik.nl"]
9
+ spec.authors = ["Julik Tarkhanov", "Andrei Horak"]
10
+ spec.email = ["me@julik.nl", "linkyndy@gmail.com"]
11
11
 
12
12
  spec.summary = %q{Process jobs from SQS}
13
13
  spec.description = %q{A full-featured library for all them SQS worker needs}
@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
36
36
  spec.add_runtime_dependency "retriable"
37
37
 
38
38
  spec.add_development_dependency "bundler"
39
- spec.add_development_dependency "rake", "~> 12.3"
39
+ spec.add_development_dependency "rake", "~> 13.0"
40
40
  spec.add_development_dependency "rspec", "~> 3.0"
41
41
 
42
42
  # The Rails deps can be relaxed, they are specified more exactly in the gemfiles/
metadata CHANGED
@@ -1,14 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sqewer
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.2.2
4
+ version: 6.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Julik Tarkhanov
8
+ - Andrei Horak
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2019-10-08 00:00:00.000000000 Z
12
+ date: 2021-02-18 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: aws-sdk-sqs
@@ -100,14 +101,14 @@ dependencies:
100
101
  requirements:
101
102
  - - "~>"
102
103
  - !ruby/object:Gem::Version
103
- version: '12.3'
104
+ version: '13.0'
104
105
  type: :development
105
106
  prerelease: false
106
107
  version_requirements: !ruby/object:Gem::Requirement
107
108
  requirements:
108
109
  - - "~>"
109
110
  - !ruby/object:Gem::Version
110
- version: '12.3'
111
+ version: '13.0'
111
112
  - !ruby/object:Gem::Dependency
112
113
  name: rspec
113
114
  requirement: !ruby/object:Gem::Requirement
@@ -223,6 +224,7 @@ dependencies:
223
224
  description: A full-featured library for all them SQS worker needs
224
225
  email:
225
226
  - me@julik.nl
227
+ - linkyndy@gmail.com
226
228
  executables:
227
229
  - sqewer
228
230
  - sqewer_rails