rocketjob 6.1.0 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +6 -6
  3. data/lib/rocket_job/batch/io.rb +2 -0
  4. data/lib/rocket_job/config.rb +1 -0
  5. data/lib/rocket_job/extensions/mongo/logging.rb +4 -0
  6. data/lib/rocket_job/jobs/copy_file_job.rb +4 -1
  7. data/lib/rocket_job/jobs/dirmon_job.rb +4 -0
  8. data/lib/rocket_job/plugins/retry.rb +55 -5
  9. data/lib/rocket_job/rocket_job.rb +3 -3
  10. data/lib/rocket_job/sliced/output.rb +1 -0
  11. data/lib/rocket_job/sliced/slices.rb +1 -1
  12. data/lib/rocket_job/version.rb +1 -1
  13. data/lib/rocketjob.rb +1 -1
  14. metadata +16 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fc1f3d316be0aa058cee5b22c116872a80ea5bdaf7d72db447ae13781ba6b370
4
- data.tar.gz: 87f56a455792e45ab2ad706f0c8bd625590df9942806bbd308c5145d07e94973
3
+ metadata.gz: d12a5469219af4b5781c67cf65670879f1fa349e42509431e5924015a52b4542
4
+ data.tar.gz: 745bbcfb5cc47a3413edf874b088b81382ccc185218aac3648adb1250e51b803
5
5
  SHA512:
6
- metadata.gz: f928a3afb886c39893ef5b5d2e13f71405c83df53c3b02826e474326c9171f87c84e595fc179a0e22cbc69d846f1da12eb268026fc06bcaf609920326159c6f4
7
- data.tar.gz: 3b90508c45e7a09739ae53e5bb66992d80251a5b5668eb568eeba30a8da8d109bbb162b2db66e93f82ce3deb4f6cc9425175d0f9c41adb351f6c3885768db3dc
6
+ metadata.gz: 012b3adb126e452c3ab4409d38d1fe02499cfc5049d5cc30074a1f607245e8da1327ad1a0a4b738ec290bcaf233cb7388132828d6f8f1684bdd70a1e17eb7c55
7
+ data.tar.gz: f8a1899798671265395f72d259afce338bea6e555755c33fdfa11928e10b14f29cc868570caa39f091f6863fa3c8809ab8874a6ba81b8b210c36c63e4454cf2b
data/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  # Rocket Job
2
- [![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Support](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
2
+ [![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
3
3
 
4
4
  Ruby's missing batch system
5
5
 
@@ -14,7 +14,7 @@ Checkout https://rocketjob.io/
14
14
 
15
15
  ## Support
16
16
 
17
- * Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
17
+ * Ask questions in [Rocket Job Discussions](https://github.com/reidmorrison/rocketjob/discussions)
18
18
  * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
19
19
 
20
20
  ## Rocket Job v6
@@ -106,8 +106,8 @@ These fields have been removed from the job itself:
106
106
  ~~~ruby
107
107
  class MyJob < RocketJob::Job
108
108
  include RocketJob::Batch
109
-
110
- self.slice_sice = 1_000
109
+
110
+ self.slice_size = 1_000
111
111
  self.encrypt = true
112
112
  self.compress = true
113
113
  end
@@ -123,8 +123,8 @@ If the serializer is set to `encrypt` then it is automatically compressed.
123
123
  ~~~ruby
124
124
  class MyJob < RocketJob::Job
125
125
  include RocketJob::Batch
126
-
127
- input_category slice_sice: 1_000, serializer: :encrypt
126
+
127
+ input_category slice_size: 1_000, serializer: :encrypt
128
128
  end
129
129
  ~~~
130
130
 
data/lib/rocket_job/batch/io.rb CHANGED
@@ -466,6 +466,8 @@ module RocketJob
466
466
  output_collection.download(header_line: binary_header_line) { |record| io.write(record) }
467
467
  end
468
468
  else
469
+ # TODO: Add category to named tags to aid problem determination
470
+ # And RJ Download metric with duration
469
471
  IOStreams.new(stream || category.file_name).writer(:line, **args) do |io|
470
472
  output_collection.download(header_line: header_line) { |record| io << record }
471
473
  end
data/lib/rocket_job/config.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require "logger/severity"
1
2
  require "yaml"
2
3
  module RocketJob
3
4
  # Rocket Job Configuration
data/lib/rocket_job/extensions/mongo/logging.rb CHANGED
@@ -6,22 +6,26 @@ module Mongo
6
6
  include SemanticLogger::Loggable
7
7
  logger.name = "Mongo"
8
8
 
9
+ undef :started
9
10
  def started(event)
10
11
  @event_command = event.command
11
12
  end
12
13
 
14
+ undef :succeeded
13
15
  def succeeded(event)
14
16
  logger.debug(message: prefix(event),
15
17
  duration: (event.duration * 1000),
16
18
  payload: @event_command)
17
19
  end
18
20
 
21
+ undef :failed
19
22
  def failed(event)
20
23
  logger.debug(message: "#{prefix(event)} Failed: #{event.message}",
21
24
  duration: (event.duration * 1000),
22
25
  payload: @event_command)
23
26
  end
24
27
 
28
+ undef :prefix
25
29
  def prefix(event)
26
30
  "#{event.address} | #{event.database_name}.#{event.command_name}"
27
31
  end
data/lib/rocket_job/jobs/copy_file_job.rb CHANGED
@@ -82,7 +82,10 @@ module RocketJob
82
82
  end
83
83
 
84
84
  def apply_streams(path, streams)
85
- streams.each_pair { |stream, args| path.stream(stream.to_sym, args.nil? ? {} : decode_args(args)) }
85
+ streams.each_pair do |stream, args|
86
+ stream_args = args.nil? ? {} : decode_args(args)
87
+ path.stream(stream.to_sym, **stream_args)
88
+ end
86
89
  end
87
90
 
88
91
  def decode_args(args)
data/lib/rocket_job/jobs/dirmon_job.rb CHANGED
@@ -42,6 +42,10 @@ module RocketJob
42
42
  self.description = "Directory Monitor"
43
43
  self.priority = 30
44
44
 
45
+ # Prevent multiple dirmon jobs from running at the same time by only
46
+ # creating the next instance when this one finishes.
47
+ self.cron_after_start = false
48
+
45
49
  # Hash[file_name, size]
46
50
  field :previous_file_names, type: Hash, default: {}, copy_on_restart: true
47
51
 
data/lib/rocket_job/plugins/retry.rb CHANGED
@@ -26,14 +26,51 @@ module RocketJob
26
26
  # # Disable retries for this job instance.
27
27
  # MyCronJob.create!(retry_limit: 0)
28
28
  #
29
+ # Below is a table of the delay for each retry attempt, as well as the _total_ duration spent retrying a job for
30
+ # the specified number of retries, excluding actual processing time.
31
+ #
32
+ # |---------|------------|----------------|
33
+ # | Attempt | Delay | Total Duration |
34
+ # |---------|------------|----------------|
35
+ # | 01 | 6.000s | 6.000s |
36
+ # | 02 | 21.000s | 27.000s |
37
+ # | 03 | 1m 26s | 1m 53s |
38
+ # | 04 | 4m 21s | 6m 14s |
39
+ # | 05 | 10m 30s | 16m 44s |
40
+ # | 06 | 21m 41s | 38m 25s |
41
+ # | 07 | 40m 6s | 20h 18m |
42
+ # | 08 | 20h 8m | 21h 26m |
43
+ # | 09 | 20h 49m | 23h 16m |
44
+ # | 10 | 21h 46m | 2h 3m |
45
+ # | 11 | 23h 4m | 6h 7m |
46
+ # | 12 | 0h 45m | 11h 52m |
47
+ # | 13 | 2h 56m | 1d 19h 48m |
48
+ # | 14 | 5h 40m | 1d 6h 29m |
49
+ # | 15 | 9h 3m | 2d 20h 33m |
50
+ # | 16 | 13h 12m | 2d 14h 45m |
51
+ # | 17 | 18h 12m | 3d 13h 57m |
52
+ # | 18 | 1d 0h 9m | 5d 19h 7m |
53
+ # | 19 | 1d 7h 12m | 6d 7h 19m |
54
+ # | 20 | 1d 15h 26m | 8d 3h 46m |
55
+ # | 21 | 2d 1h 1m | 10d 9h 47m |
56
+ # | 22 | 2d 12h 4m | 13d 2h 51m |
57
+ # | 23 | 3d 0h 44m | 16d 8h 35m |
58
+ # | 24 | 3d 15h 9m | 20d 4h 45m |
59
+ # | 25 | 4d 7h 30m | 24d 17h 16 |
60
+ # | 26 | 5d 1h 56m | 30d 0h 12m |
61
+ # | 27 | 6d 22h 37m | 36d 3h 49m |
62
+ # | 28 | 7d 21h 44m | 43d 6h 34m |
63
+ # | 29 | 8d 23h 28m | 51d 11h 2m |
64
+ # | 30 | 9d 4h 0m | 61d 20h 2m |
65
+ # |---------|------------|----------------|
29
66
  module Retry
30
67
  extend ActiveSupport::Concern
31
68
 
32
69
  included do
33
70
  after_fail :rocket_job_retry
34
71
 
35
- # Maximum number of times to retry this job
36
- # 25 is approximately 3 weeks of retries
72
+ # Maximum number of times to retry this job.
73
+ # The default of 25 is almost 25 days of retries.
37
74
  field :retry_limit, type: Integer, default: 25, class_attribute: true, user_editable: true, copy_on_restart: true
38
75
 
39
76
  # List of times when this job failed
@@ -73,10 +110,23 @@ module RocketJob
73
110
  self.worker_name = nil
74
111
  end
75
112
 
76
- # Returns [Time] when to retry this job at
77
- # Same basic formula as Delayed Job
113
+ # Returns [Integer] the number of seconds after which to retry this failed job.
114
+ # Uses an exponential back-off algorithm to prevent overloading the failed resource.
115
+ #
116
+ # For example, to see the durations for the first 25 retries:
117
+ # count = 25
118
+ # intervals = (1..count).map { |attempt| attempt**4 + 5 }
119
+ #
120
+ # Display the above intervals as human readable durations:
121
+ # intervals.map { |seconds| RocketJob.seconds_as_duration(seconds) }
122
+ #
123
+ # Then sum the total duration in seconds:
124
+ # RocketJob.seconds_as_duration(intervals.sum)
125
+ #
126
+ # Or, to see the total durations based on the number of retries:
127
+ # (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
78
128
  def rocket_job_retry_seconds_to_delay
79
- (rocket_job_failure_count**4) + 15 + (rand(30) * (rocket_job_failure_count + 1))
129
+ rocket_job_failure_count**4 + 5
80
130
  end
81
131
  end
82
132
  end
data/lib/rocket_job/rocket_job.rb CHANGED
@@ -28,11 +28,11 @@ module RocketJob
28
28
  return nil unless seconds
29
29
 
30
30
  if seconds >= 86_400.0 # 1 day
31
- "#{(seconds / 86_400).to_i}d #{Time.at(seconds).strftime('%-Hh %-Mm')}"
31
+ "#{(seconds / 86_400).to_i}d #{Time.at(seconds).utc.strftime('%-Hh %-Mm')}"
32
32
  elsif seconds >= 3600.0 # 1 hour
33
- Time.at(seconds).strftime("%-Hh %-Mm")
33
+ Time.at(seconds).utc.strftime("%-Hh %-Mm")
34
34
  elsif seconds >= 60.0 # 1 minute
35
- Time.at(seconds).strftime("%-Mm %-Ss")
35
+ Time.at(seconds).utc.strftime("%-Mm %-Ss")
36
36
  elsif seconds >= 1.0 # 1 second
37
37
  format("%.3fs", seconds)
38
38
  else
data/lib/rocket_job/sliced/output.rb CHANGED
@@ -12,6 +12,7 @@ module RocketJob
12
12
  # Call the supplied block for every record returned
13
13
  record_count = 0
14
14
  each do |slice|
15
+ # TODO: Add slice_id to named tags to aid problem determination
15
16
  slice.each do |record|
16
17
  record_count += 1
17
18
  yield(record)
data/lib/rocket_job/sliced/slices.rb CHANGED
@@ -92,7 +92,7 @@ module RocketJob
92
92
 
93
93
  def insert_many(slices)
94
94
  documents = slices.collect(&:as_document)
95
- all.collection.insert_many(documents)
95
+ all.collection.insert_many(documents) if documents.present?
96
96
  end
97
97
 
98
98
  # Append to an existing slice if already present
data/lib/rocket_job/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module RocketJob
2
- VERSION = "6.1.0".freeze
2
+ VERSION = "6.2.0".freeze
3
3
  end
data/lib/rocketjob.rb CHANGED
@@ -1,6 +1,6 @@
1
- require "iostreams"
2
1
  require "semantic_logger"
3
2
  require "symmetric-encryption"
3
+ require "iostreams"
4
4
  require "mongoid"
5
5
  require "rocket_job/extensions/mongo/logging"
6
6
  require "rocket_job/extensions/iostreams/path"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rocketjob
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.1.0
4
+ version: 6.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Reid Morrison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-11-04 00:00:00.000000000 Z
11
+ date: 2023-07-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aasm
@@ -28,58 +28,58 @@ dependencies:
28
28
  name: concurrent-ruby
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - "~>"
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '1.1'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - "~>"
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.1'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: fugit
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '1.4'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '1.4'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: iostreams
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "~>"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '1.9'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "~>"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '1.9'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: mongoid
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - "~>"
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
- version: '7.1'
75
+ version: '7.5'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - "~>"
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
- version: '7.1'
82
+ version: '7.5'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: semantic_logger
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -98,14 +98,14 @@ dependencies:
98
98
  name: symmetric-encryption
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - "~>"
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: '4.3'
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - "~>"
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '4.3'
111
111
  description:
@@ -229,14 +229,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
229
229
  requirements:
230
230
  - - ">="
231
231
  - !ruby/object:Gem::Version
232
- version: '2.5'
232
+ version: '2.7'
233
233
  required_rubygems_version: !ruby/object:Gem::Requirement
234
234
  requirements:
235
235
  - - ">="
236
236
  - !ruby/object:Gem::Version
237
237
  version: '0'
238
238
  requirements: []
239
- rubygems_version: 3.2.22
239
+ rubygems_version: 3.4.9
240
240
  signing_key:
241
241
  specification_version: 4
242
242
  summary: Ruby's missing batch processing system.