sqewer 6.2.2 → 6.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 70857d657ec0e0190310f806f8b34a8c04df7504c67e9890cd2ee2580e5ef387
4
- data.tar.gz: e3d7d1fa311e25ac8e53b58db842446cff851924cba36583bc9bf87a9a5120b9
3
+ metadata.gz: cf1c2d6717fda50ce8a72721d00ffcfd2fae867f40f40a07a0cf48cbe94fe17b
4
+ data.tar.gz: fb1622a03a4f540aaa96d24dc3d5d39c5d400b658f371c88dc682844ac784ea5
5
5
  SHA512:
6
- metadata.gz: ba846399144de508eff227a61e7d78e6fa662884565670183f9992530e06790b704c013b866436fec0f9f860872908a528db6db8751a711811480b9f9f7b0e5f
7
- data.tar.gz: 708a29a0e29253143002c639a963dc65e966641045113c4a826a84536935969f5906da291d865042e3655da5399eb5d9e95805d96d306462bf5dd2266f924dd7
6
+ metadata.gz: fd4e73838959531d646d0ed69fd8316e85075d3abe96b5362c58ff5de9af27b26cec142273bce674aaadf571275f1f91cf055c305be4b9797edd75eb89b74222
7
+ data.tar.gz: 9f4b9006f504bd4a0a85abea9d3c82bf2c70ccdfecf59095df58a1e6b1b8b3ed0b6ba819778f828c48e5dd1865c1f312c5feda78485e9ceab95565e9559cc870
data/.gitignore CHANGED
@@ -1,54 +1,56 @@
1
- .rspec
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+
13
+ # Used by dotenv library to load environment variables.
2
14
  .env
3
15
 
4
- # rcov generated
5
- coverage
6
- coverage.data
16
+ # Ignore Byebug command history file.
17
+ .byebug_history
7
18
 
8
- # rdoc generated
9
- rdoc
19
+ ## Specific to RubyMotion:
20
+ .dat*
21
+ .repl_history
22
+ build/
23
+ *.bridgesupport
24
+ build-iPhoneOS/
25
+ build-iPhoneSimulator/
10
26
 
11
- # yard generated
12
- doc
13
- .yardoc
14
-
15
- # bundler
16
- .bundle
17
- Gemfile.lock
18
- gemfiles/*.lock
19
-
20
- # jeweler generated
21
- pkg
22
-
23
- # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore:
27
+ ## Specific to RubyMotion (use of CocoaPods):
24
28
  #
25
- # * Create a file at ~/.gitignore
26
- # * Include files you want ignored
27
- # * Run: git config --global core.excludesfile ~/.gitignore
29
+ # We recommend against adding the Pods directory to your .gitignore. However
30
+ # you should judge for yourself, the pros and cons are mentioned at:
31
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
28
32
  #
29
- # After doing this, these files will be ignored in all your git projects,
30
- # saving you from having to 'pollute' every project you touch with them
31
- #
32
- # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
33
- #
34
- # For MacOS:
35
- #
36
- #.DS_Store
33
+ # vendor/Pods/
37
34
 
38
- # For TextMate
39
- #*.tmproj
40
- #tmtags
35
+ ## Documentation cache and generated files:
36
+ /.yardoc/
37
+ /_yardoc/
38
+ /doc/
39
+ /rdoc/
41
40
 
42
- # For emacs:
43
- #*~
44
- #\#*
45
- #.\#*
41
+ ## Environment normalization:
42
+ /.bundle/
43
+ /vendor/bundle
44
+ /lib/bundler/man/
46
45
 
47
- # For vim:
48
- #*.swp
46
+ # for a library or gem, you might want to ignore these files since the code is
47
+ # intended to run in multiple environments; otherwise, check them in:
48
+ Gemfile.lock
49
+ .ruby-version
50
+ .ruby-gemset
49
51
 
50
- # For redcar:
51
- #.redcar
52
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
53
+ .rvmrc
52
54
 
53
- # For rubinius:
54
- #*.rbc
55
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
56
+ # .rubocop-https?--*
data/.travis.yml CHANGED
@@ -6,6 +6,8 @@ rvm:
6
6
  - 2.3.7
7
7
  - 2.4.5
8
8
  - 2.5.1
9
+ - 2.6.5
10
+ - 2.7.0
9
11
  cache: bundler
10
12
  sudo: false
11
13
  env:
@@ -19,3 +21,7 @@ matrix:
19
21
  gemfile: gemfiles/Gemfile.rails-5.1.x
20
22
  - rvm: 2.5.1
21
23
  gemfile: gemfiles/Gemfile.rails-4.2.x
24
+ - rvm: 2.6.5
25
+ gemfile: gemfiles/Gemfile.rails-4.2.x
26
+ - rvm: 2.7.0
27
+ gemfile: gemfiles/Gemfile.rails-4.2.x
data/CHANGELOG.md CHANGED
@@ -1,3 +1,23 @@
1
+ ### 6.5.1
2
+ - Also retry on `Aws::SQS::Errors::InternalError` exception when receiving/sending messages. This will make
3
+ the receiving thread more resilient to sudden SQS failures. By the time SQS recovers the receiving thread
4
+ should stay alive.
5
+
6
+ ### 6.5.0
7
+ - Adds `$stdout.sync = true` to CLI to flush the logs to STDOUT
8
+
9
+ ### 6.4.1
10
+ - Retrieve attributes when receiving messages from SQS
11
+
12
+ ### 6.4.0
13
+ - Raise an exception in submit! if the job serializes to a message that is
14
+ above the native SQS limit for message size.
15
+ - Ensure SendMessageBatch is only performed for batches totaling 256KB of message size or less.
16
+ - Insert Sqewer::Error between StandardError and our custom errors for easier rescuing
17
+
18
+ ### 6.3.0
19
+ - Add support for Ruby 2.7
20
+
1
21
  ### 6.2.2
2
22
  - Test the Appsignal integration using actual Appsignal libraries
3
23
  - In the Appsignal integration, replace a call of `set_queue_start=` with `set_queue_start`
data/README.md CHANGED
@@ -20,14 +20,14 @@ and to start processing, in your commandline handler:
20
20
 
21
21
  #!/usr/bin/env ruby
22
22
  require 'my_applicaion'
23
- Sqewer::CLI.run
23
+ Sqewer::CLI.start
24
24
 
25
25
  To add arguments to the job
26
26
 
27
27
  class JobWithArgs
28
28
  include Sqewer::SimpleJob
29
29
  attr_accessor :times
30
-
30
+
31
31
  def run
32
32
  ...
33
33
  end
@@ -88,6 +88,10 @@ Note that at this point only arguments that are raw JSON types are supported:
88
88
 
89
89
  If you need marshalable Ruby types there instead, you might need to implement a custom `Serializer.`
90
90
 
91
+ ### Sqewer::SimpleJob
92
+
93
+ The module `Sqewer::SimpleJob` can be included to a job class to add some features, specially dealing with attributes, see more details [here](https://github.com/WeTransfer/sqewer/blob/master/lib/sqewer/simple_job.rb).
94
+
91
95
  ## Jobs spawning dependent jobs
92
96
 
93
97
  If your `run` method on the job object accepts arguments (has non-zero `arity` ) the `ExecutionContext` will
@@ -119,11 +123,11 @@ include all the keyword arguments needed to instantiate the job when executing.
119
123
  def initialize(to:, body:)
120
124
  ...
121
125
  end
122
-
126
+
123
127
  def run()
124
128
  ...
125
129
  end
126
-
130
+
127
131
  def to_h
128
132
  {to: @to, body: @body}
129
133
  end
@@ -151,7 +155,7 @@ conform to the job serialization format used internally. For example, you can ha
151
155
  message = JSON.load(message_blob)
152
156
  return if message['Service'] # AWS test
153
157
  return HandleS3Notification.new(message) if message['Records']
154
-
158
+
155
159
  super # as default
156
160
  end
157
161
  end
@@ -176,7 +180,7 @@ The very minimal executable for running jobs would be this:
176
180
 
177
181
  #!/usr/bin/env ruby
178
182
  require 'my_applicaion'
179
- Sqewer::CLI.run
183
+ Sqewer::CLI.start
180
184
 
181
185
  This will connect to the queue at the URL set in the `SQS_QUEUE_URL` environment variable, and
182
186
  use all the default parameters. The `CLI` module will also set up a signal handler to terminate
@@ -233,9 +237,9 @@ you generate. For example, you could use a pipe. But in a more general case some
233
237
  ActiveRAMGobbler.fetch_stupendously_many_things.each do |...|
234
238
  end
235
239
  end
236
-
240
+
237
241
  _, status = Process.wait2(pid)
238
-
242
+
239
243
  # Raise an error in the parent process to signal Sqewer that the job failed
240
244
  # if the child exited with a non-0 status
241
245
  raise "Child process crashed" unless status.exitstatus && status.exitstatus.zero?
@@ -252,7 +256,7 @@ You can wrap job processing in middleware. A full-featured middleware class look
252
256
  # msg_id is the receipt handle, msg_payload is the message body string, msg_attributes are the message's attributes
253
257
  yield
254
258
  end
255
-
259
+
256
260
  # Surrounds the actual job execution
257
261
  def around_execution(job, context)
258
262
  # job is the actual job you will be running, context is the ExecutionContext.
@@ -378,7 +382,7 @@ products using this library, was a very bad idea (more workload for deployment).
378
382
 
379
383
  ## Why so many configurable components?
380
384
 
381
- Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your
385
+ Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your
382
386
  implementation in instead. One product needs foreign-submitted SQS jobs (S3 notifications). Another product
383
387
  needs a custom Logger subclass. Yet another product needs process-based concurrency on top of threads.
384
388
  Yet another process needs to manage database connections when running the jobs. Have 3-4 of those, and a
@@ -398,7 +402,7 @@ Because I found that a producer-consumer model with a thread pool works quite we
398
402
  the Ruby standard library alone.
399
403
 
400
404
  ## Contributing to the library
401
-
405
+
402
406
  * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
403
407
  * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
404
408
  * Fork the project.
@@ -4,5 +4,5 @@ source "http://rubygems.org"
4
4
  gemspec path: __dir__ + '/..'
5
5
 
6
6
  gem 'sqlite3', '~> 1.3.6'
7
- gem 'activejob', "~> 4.2"
8
- gem 'activerecord', "~> 4.2"
7
+ gem 'activejob', "~> 4.2.0"
8
+ gem 'activerecord', "~> 4.2.0"
@@ -4,5 +4,5 @@ source "http://rubygems.org"
4
4
  gemspec path: __dir__ + '/..'
5
5
 
6
6
  gem 'sqlite3', '~> 1.3.6'
7
- gem 'activejob', "~> 5", "< 5.1"
8
- gem 'activerecord', "~> 5", "< 5.1"
7
+ gem 'activejob', "~> 5.0.0"
8
+ gem 'activerecord', "~> 5.0.0"
@@ -3,6 +3,6 @@ source "http://rubygems.org"
3
3
  # Gemspec as base dependency set
4
4
  gemspec path: __dir__ + '/..'
5
5
 
6
- gem 'sqlite3', '~> 1.3.6'
7
- gem 'activejob', "~> 5.1", "< 5.2"
8
- gem 'activerecord', "~> 5.1", "< 5.2"
6
+ gem 'sqlite3', "~> 1.3", ">= 1.3.6"
7
+ gem 'activejob', "~> 5.1.0"
8
+ gem 'activerecord', "~> 5.1.0"
data/lib/sqewer.rb CHANGED
@@ -1,5 +1,8 @@
1
1
  # The enclosing module for the library
2
2
  module Sqewer
3
+ class Error < StandardError
4
+ end
5
+
3
6
  # Eager-load everything except extensions. Sort to ensure
4
7
  # the files load in the same order on all platforms.
5
8
  Dir.glob(__dir__ + '/**/*.rb').sort.each do |path|
@@ -23,7 +26,7 @@ module Sqewer
23
26
  def self.submit!(*jobs, **options)
24
27
  Sqewer::Submitter.default.submit!(*jobs, **options)
25
28
  end
26
-
29
+
27
30
  # If we are within Rails, load the railtie
28
31
  require_relative 'sqewer/extensions/railtie' if defined?(Rails)
29
32
 
data/lib/sqewer/cli.rb CHANGED
@@ -1,3 +1,4 @@
1
+ $stdout.sync = true
1
2
  # Wraps a Worker object in a process-wide commanline handler. Once the `start` method is
2
3
  # called, signal handlers will be installed for the following signals:
3
4
  #
@@ -11,7 +11,7 @@ class Sqewer::Connection
11
11
  MAX_RANDOM_FAILURES_PER_CALL = 10
12
12
  MAX_RANDOM_RECEIVE_FAILURES = 100 # sure to hit the max_elapsed_time of 900 seconds
13
13
 
14
- NotOurFaultAwsError = Class.new(StandardError)
14
+ NotOurFaultAwsError = Class.new(Sqewer::Error)
15
15
 
16
16
  # A wrapper for most important properties of the received message
17
17
  class Message < Struct.new(:receipt_handle, :body, :attributes)
@@ -52,11 +52,15 @@ class Sqewer::Connection
52
52
  # even after 15 minutes it is either down or the server is misconfigured. Either way it makes no sense to
53
53
  # continue.
54
54
  #
55
- # @return [Array<Message>] an array of Message objects
55
+ # @return [Array<Message>] an array of Message objects
56
56
  def receive_messages
57
- Retriable.retriable on: Seahorse::Client::NetworkingError, tries: MAX_RANDOM_RECEIVE_FAILURES do
58
- response = client.receive_message(queue_url: @queue_url,
59
- wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: BATCH_RECEIVE_SIZE)
57
+ Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_RECEIVE_FAILURES do
58
+ response = client.receive_message(
59
+ queue_url: @queue_url,
60
+ attribute_names: ['All'],
61
+ wait_time_seconds: DEFAULT_TIMEOUT_SECONDS,
62
+ max_number_of_messages: BATCH_RECEIVE_SIZE
63
+ )
60
64
  response.messages.map {|message| Message.new(message.receipt_handle, message.body, message.attributes) }
61
65
  end
62
66
  end
@@ -65,7 +69,7 @@ class Sqewer::Connection
65
69
  #
66
70
  # @param message_body[String] the message to send
67
71
  # @param kwargs_for_send[Hash] additional arguments for the submit (such as `delay_seconds`).
68
- # Passes the arguments to the AWS SDK.
72
+ # Passes the arguments to the AWS SDK.
69
73
  # @return [void]
70
74
  def send_message(message_body, **kwargs_for_send)
71
75
  send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
@@ -91,6 +95,58 @@ class Sqewer::Connection
91
95
  m[:delay_seconds] = kwargs_for_send[:delay_seconds] if kwargs_for_send[:delay_seconds]
92
96
  messages << m
93
97
  end
98
+
99
+ # each_batch here also needs to ensure that the sum of payload lengths does not exceed 256kb
100
+ def each_batch
101
+ regrouped = pack_into_batches(messages, weight_limit: 256 * 1024, batch_length_limit: 10) do |message|
102
+ message.fetch(:message_body).bytesize
103
+ end
104
+ regrouped.each { |b| yield(b) }
105
+ end
106
+
107
+ # Optimizes a large list of items into batches of 10 items
108
+ # or less and with the sum of item lengths being below 256KB
109
+ # The block given to the method should return the weight of the given item
110
+ def pack_into_batches(items, weight_limit:, batch_length_limit:)
111
+ batches = []
112
+ current_batch = []
113
+ current_batch_weight = 0
114
+
115
+ # Sort the items by their weight (length of the body).
116
+ sorted_items = items.sort_by { |item| yield(item) }
117
+
118
+ # and then take 1 item from the list and append it to the batch if it fits.
119
+ # If it doesn't fit, no item after it will fit into this batch either (!)
120
+ # which is how we can optimize
121
+ sorted_items.each_with_index do |item|
122
+ weight_of_this_item = yield(item)
123
+
124
+ # First protect from invalid input
125
+ if weight_of_this_item > weight_limit
126
+ raise "#{item.inspect} was larger than the permissible limit"
127
+ # The first limit is on the item count per batch -
128
+ # if we are limited on that the batch needs to be closed
129
+ elsif current_batch.length == batch_length_limit
130
+ batches << current_batch
131
+ current_batch = []
132
+ current_batch_weight = 0
133
+ # If placing this item in the batch would make the batch overweight
134
+ # we need to close the batch, because all the items which come after
135
+ # this one will be same size or larger. This is the key part of the optimization.
136
+ elsif (current_batch_weight + weight_of_this_item) > weight_limit
137
+ batches << current_batch
138
+ current_batch = []
139
+ current_batch_weight = 0
140
+ end
141
+
142
+ # and then append the item to the current batch
143
+ current_batch_weight += weight_of_this_item
144
+ current_batch << item
145
+ end
146
+ batches << current_batch unless current_batch.empty?
147
+
148
+ batches
149
+ end
94
150
  end
95
151
 
96
152
  # Saves the receipt handles to batch-delete from the SQS queue
@@ -135,8 +191,12 @@ class Sqewer::Connection
135
191
 
136
192
  private
137
193
 
194
+ def network_and_aws_sdk_errors
195
+ [NotOurFaultAwsError, Seahorse::Client::NetworkingError, Aws::SQS::Errors::InternalError]
196
+ end
197
+
138
198
  def handle_batch_with_retries(method, batch)
139
- Retriable.retriable on: [NotOurFaultAwsError, Seahorse::Client::NetworkingError], tries: MAX_RANDOM_FAILURES_PER_CALL do
199
+ Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_FAILURES_PER_CALL do
140
200
  resp = client.send(method, queue_url: @queue_url, entries: batch)
141
201
  wrong_messages, aws_failures = resp.failed.partition {|m| m.sender_fault }
142
202
  if wrong_messages.any?
@@ -4,7 +4,6 @@ module Sqewer
4
4
  # to Appsignal and to monitor performance. Will only activate
5
5
  # if the Appsignal gem is loaded within the current process and active.
6
6
  class AppsignalWrapper
7
-
8
7
  def self.new
9
8
  if defined?(Appsignal)
10
9
  super
@@ -13,7 +13,7 @@ class Sqewer::Serializer
13
13
  @instance ||= new
14
14
  end
15
15
 
16
- AnonymousJobClass = Class.new(StandardError)
16
+ AnonymousJobClass = Class.new(Sqewer::Error)
17
17
 
18
18
  # Instantiate a Job object from a message body string. If the
19
19
  # returned result is `nil`, the job will be skipped.
@@ -33,18 +33,18 @@ class Sqewer::Serializer
33
33
  # use a default that will put us ahead of that execution deadline from the start.
34
34
  t = Time.now.to_i
35
35
  execute_after = job_ticket_hash.fetch(:_execute_after) { t - 5 }
36
-
36
+
37
37
  job_params = job_ticket_hash.delete(:_job_params)
38
38
  job = if job_params.nil? || job_params.empty?
39
39
  job_class.new # no args
40
40
  else
41
41
  job_class.new(**job_params) # The rest of the message are keyword arguments for the job
42
42
  end
43
-
44
- # If the job is not up for execution now, wrap it with something that will
43
+
44
+ # If the job is not up for execution now, wrap it with something that will
45
45
  # re-submit it for later execution when the run() method is called
46
46
  return ::Sqewer::Resubmit.new(job, execute_after) if execute_after > t
47
-
47
+
48
48
  job
49
49
  end
50
50
 
@@ -65,7 +65,7 @@ class Sqewer::Serializer
65
65
  job_params = job.respond_to?(:to_h) ? job.to_h : nil
66
66
  job_ticket_hash = {_job_class: job_class_name, _job_params: job_params}
67
67
  job_ticket_hash[:_execute_after] = execute_after_timestamp.to_i if execute_after_timestamp
68
-
68
+
69
69
  JSON.dump(job_ticket_hash)
70
70
  end
71
71
  end
@@ -4,9 +4,16 @@
4
4
  # * initialize() will have keyword access to all accessors, and will ensure you have called each one of them
5
5
  # * to_h() will produce a symbolized Hash with all the properties defined using attr_accessor, and the job_class_name
6
6
  # * inspect() will provide a sensible default string representation for logging
7
+ #
8
+ # This module validates if the attributes defined in the job class are the same as
9
+ # those persisted in the queue. More details on `Sqewer::SimpleJob#initialize`.
10
+ # Because of this, it's required to create a new job class when adding or removing
11
+ # an attribute.
12
+ # This mechanism guarantees strong consistency. Without it, a new deployed job class
13
+ # could process old incompatible payloads.
7
14
  module Sqewer::SimpleJob
8
- UnknownJobAttribute = Class.new(StandardError)
9
- MissingAttribute = Class.new(StandardError)
15
+ UnknownJobAttribute = Class.new(Sqewer::Error)
16
+ MissingAttribute = Class.new(Sqewer::Error)
10
17
 
11
18
  EQ_END = /(\w+)(\=)$/
12
19
 
@@ -53,7 +60,7 @@ module Sqewer::SimpleJob
53
60
  accessor = "#{k}="
54
61
  touched_attributes << k
55
62
  unless respond_to?(accessor)
56
- raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}"
63
+ raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}"
57
64
  end
58
65
 
59
66
  send("#{k}=", v)
@@ -3,7 +3,10 @@
3
3
  # and the serializer (something that responds to `#serialize`) to
4
4
  # convert the job into the string that will be put in the queue.
5
5
  class Sqewer::Submitter < Struct.new(:connection, :serializer)
6
- NotSqewerJob = Class.new(StandardError)
6
+ MAX_PERMITTED_MESSAGE_SIZE_BYTES = 256 * 1024
7
+
8
+ NotSqewerJob = Class.new(Sqewer::Error)
9
+ MessageTooLarge = Class.new(Sqewer::Error)
7
10
 
8
11
  # Returns a default Submitter, configured with the default connection
9
12
  # and the default serializer.
@@ -12,7 +15,7 @@ class Sqewer::Submitter < Struct.new(:connection, :serializer)
12
15
  end
13
16
 
14
17
  def submit!(job, **kwargs_for_send)
15
- raise NotSqewerJob.new("Submitted object is not a valid job: #{job.inspect}") unless job.respond_to?(:run)
18
+ validate_job_responds_to_run!(job)
16
19
  message_body = if delay_by_seconds = kwargs_for_send[:delay_seconds]
17
20
  clamped_delay = clamp_delay(delay_by_seconds)
18
21
  kwargs_for_send[:delay_seconds] = clamped_delay
@@ -21,11 +24,26 @@ class Sqewer::Submitter < Struct.new(:connection, :serializer)
21
24
  else
22
25
  serializer.serialize(job)
23
26
  end
27
+ validate_message_for_size!(message_body, job)
28
+
24
29
  connection.send_message(message_body, **kwargs_for_send)
25
30
  end
26
-
31
+
27
32
  private
28
-
33
+
34
+ def validate_job_responds_to_run!(job)
35
+ return if job.respond_to?(:run)
36
+ error_message = "Submitted object is not a valid job (does not respond to #run): #{job.inspect}"
37
+ raise NotSqewerJob.new(error_message)
38
+ end
39
+
40
+ def validate_message_for_size!(message_body, job)
41
+ actual_bytesize = message_body.bytesize
42
+ return if actual_bytesize <= MAX_PERMITTED_MESSAGE_SIZE_BYTES
43
+ error_message = "Job #{job.inspect} serialized to a message which was too large (#{actual_bytesize} bytes)"
44
+ raise MessageTooLarge.new(error_message)
45
+ end
46
+
29
47
  def clamp_delay(delay)
30
48
  [1, 899, delay].sort[1]
31
49
  end
@@ -1,3 +1,3 @@
1
1
  module Sqewer
2
- VERSION = '6.2.2'
2
+ VERSION = '6.5.1'
3
3
  end
data/lib/sqewer/worker.rb CHANGED
@@ -48,7 +48,7 @@ class Sqewer::Worker
48
48
  #
49
49
  # @param connection[Sqewer::Connection] the object that handles polling and submitting
50
50
  # @param serializer[#serialize, #unserialize] the serializer/unserializer for the jobs
51
- # @param execution_context_class[Class] the class for the execution context (will be instantiated by
51
+ # @param execution_context_class[Class] the class for the execution context (will be instantiated by
52
52
  # the worker for each job execution)
53
53
  # @param submitter_class[Class] the class used for submitting jobs (will be instantiated by the worker for each job execution)
54
54
  # @param middleware_stack[Sqewer::MiddlewareStack] the middleware stack that is going to be used
@@ -91,7 +91,7 @@ class Sqewer::Worker
91
91
 
92
92
  consumers = (1..@num_threads).map do
93
93
  Thread.new do
94
- catch(:goodbye) { loop {take_and_execute} }
94
+ loop { take_and_execute }
95
95
  end
96
96
  end
97
97
 
@@ -212,58 +212,34 @@ class Sqewer::Worker
212
212
  submitter = submitter_class.new(box, serializer)
213
213
  context = execution_context_class.new(submitter, {'logger' => logger})
214
214
 
215
- t = Time.now
215
+ t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
216
216
  middleware_stack.around_execution(job, context) do
217
217
  job.method(:run).arity.zero? ? job.run : job.run(context)
218
+ # delete_message will enqueue the message for deletion,
219
+ # but when flush! is called _first_ the messages pending will be
220
+ # delivered to the queue, THEN all the deletes are going to be performed.
221
+ # So it is safe to call delete here first - the delete won't get to the queue
222
+ # if flush! fails to spool pending messages.
223
+ box.delete_message(message.receipt_handle)
224
+ n_flushed = box.flush!
225
+ logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed > 0
218
226
  end
219
- box.delete_message(message.receipt_handle)
220
227
 
221
- delta = Time.now - t
228
+ delta = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
222
229
  logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
223
- ensure
224
- n_flushed = box.flush!
225
- logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
226
230
  end
227
231
 
228
232
  def take_and_execute
229
233
  message = @execution_queue.pop(nonblock=true)
230
234
  handle_message(message)
231
235
  rescue ThreadError # Queue is empty
232
- throw :goodbye if stopping?
233
- sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
236
+ if stopping?
237
+ Thread.current.exit
238
+ else
239
+ sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
240
+ end
234
241
  rescue => e # anything else, at or below StandardError that does not need us to quit
235
242
  @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..64], e.class, e.message] }
236
243
  e.backtrace.each { |s| @logger.debug{"\t#{s}"} }
237
244
  end
238
-
239
- def perform(message)
240
- # Create a messagebox that buffers all the calls to Connection, so that
241
- # we can send out those commands in one go (without interfering with senders
242
- # on other threads, as it seems the Aws::SQS::Client is not entirely
243
- # thread-safe - or at least not it's HTTP client part).
244
- box = Sqewer::ConnectionMessagebox.new(connection)
245
-
246
- job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body, message.attributes) do
247
- serializer.unserialize(message.body)
248
- end
249
- return unless job
250
-
251
- submitter = submitter_class.new(box, serializer)
252
- context = execution_context_class.new(submitter, {'logger' => logger})
253
-
254
- t = Time.now
255
- middleware_stack.around_execution(job, context) do
256
- job.method(:run).arity.zero? ? job.run : job.run(context)
257
- end
258
-
259
- # Perform two flushes, one for any possible jobs the job has spawned,
260
- # and one for the job delete afterwards
261
- box.delete_message(message.receipt_handle)
262
-
263
- delta = Time.now - t
264
- logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
265
- ensure
266
- n_flushed = box.flush!
267
- logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
268
- end
269
245
  end
data/sqewer.gemspec CHANGED
@@ -6,8 +6,8 @@ require "sqewer/version"
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = "sqewer"
8
8
  spec.version = Sqewer::VERSION
9
- spec.authors = ["Julik Tarkhanov"]
10
- spec.email = ["me@julik.nl"]
9
+ spec.authors = ["Julik Tarkhanov", "Andrei Horak"]
10
+ spec.email = ["me@julik.nl", "linkyndy@gmail.com"]
11
11
 
12
12
  spec.summary = %q{Process jobs from SQS}
13
13
  spec.description = %q{A full-featured library for all them SQS worker needs}
@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
36
36
  spec.add_runtime_dependency "retriable"
37
37
 
38
38
  spec.add_development_dependency "bundler"
39
- spec.add_development_dependency "rake", "~> 12.3"
39
+ spec.add_development_dependency "rake", "~> 13.0"
40
40
  spec.add_development_dependency "rspec", "~> 3.0"
41
41
 
42
42
  # The Rails deps can be relaxed, they are specified more exactly in the gemfiles/
metadata CHANGED
@@ -1,14 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sqewer
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.2.2
4
+ version: 6.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Julik Tarkhanov
8
+ - Andrei Horak
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2019-10-08 00:00:00.000000000 Z
12
+ date: 2021-02-18 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: aws-sdk-sqs
@@ -100,14 +101,14 @@ dependencies:
100
101
  requirements:
101
102
  - - "~>"
102
103
  - !ruby/object:Gem::Version
103
- version: '12.3'
104
+ version: '13.0'
104
105
  type: :development
105
106
  prerelease: false
106
107
  version_requirements: !ruby/object:Gem::Requirement
107
108
  requirements:
108
109
  - - "~>"
109
110
  - !ruby/object:Gem::Version
110
- version: '12.3'
111
+ version: '13.0'
111
112
  - !ruby/object:Gem::Dependency
112
113
  name: rspec
113
114
  requirement: !ruby/object:Gem::Requirement
@@ -223,6 +224,7 @@ dependencies:
223
224
  description: A full-featured library for all them SQS worker needs
224
225
  email:
225
226
  - me@julik.nl
227
+ - linkyndy@gmail.com
226
228
  executables:
227
229
  - sqewer
228
230
  - sqewer_rails