rocketjob 7.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,8 +29,8 @@ module RocketJob
29
29
  self.priority = 30
30
30
 
31
31
  # File names in IOStreams URL format.
32
- field :source_url, type: String, user_editable: true
33
- field :target_url, type: String, user_editable: true
32
+ field :source_url, type: String, user_editable: true, path: true
33
+ field :target_url, type: String, user_editable: true, path: true
34
34
 
35
35
  # Any optional arguments to pass through to the IOStreams source and/or target.
36
36
  field :source_args, type: Hash, default: -> { {} }, user_editable: true
@@ -51,6 +51,8 @@ module RocketJob
51
51
  validates_presence_of :source_url, unless: :source_data
52
52
  validates_presence_of :target_url
53
53
  validates_presence_of :source_data, unless: :source_url
54
+ validate :source_path_is_valid, if: -> { source_url && path_changed?(:source) }
55
+ validate :target_path_is_valid, if: -> { target_url && path_changed?(:target) }
54
56
 
55
57
  before_save :set_description
56
58
 
@@ -76,35 +78,127 @@ module RocketJob
76
78
  target
77
79
  end
78
80
 
81
+ # Returns [Hash] the attributes to show, see RocketJob::Plugins::Job::Model#display_attributes, with the secrets
82
+ # in the arguments and streams of the source and target replaced, such as an SFTP password or a PGP passphrase.
83
+ # IOStreams decides which are secret, from the kind of path of each url.
84
+ def display_attributes
85
+ attrs = super
86
+ %w[source target].each do |side|
87
+ args = attrs["#{side}_args"]
88
+ streams = attrs["#{side}_streams"]
89
+ attrs["#{side}_args"] = IOStreams.redact_path_options(self["#{side}_url"].to_s, args) if args.is_a?(Hash)
90
+ attrs["#{side}_streams"] = IOStreams.redact_stream_options(streams) if streams.is_a?(Hash)
91
+ end
92
+ attrs
93
+ end
94
+
79
95
  private
80
96
 
97
+ def source_path_is_valid
98
+ validate_path(:source)
99
+ end
100
+
101
+ def target_path_is_valid
102
+ validate_path(:target)
103
+ end
104
+
105
+ # Whether the url, arguments or streams of the source or target are new or changed, so that a job whose path
106
+ # was valid when it was created, can still be saved, for example when it fails.
107
+ def path_changed?(side)
108
+ new_record? || %w[url args streams].any? { |name| attribute_changed?("#{side}_#{name}") }
109
+ end
110
+
111
+ # Builds the path of the source or target without using it, so that IOStreams checks its url, the names and
112
+ # values of its arguments, and its streams, in the same way as #perform. Encrypted arguments are not decrypted,
113
+ # nor secrets fetched, since their values are not used.
114
+ def validate_path(side)
115
+ url = public_send("#{side}_url")
116
+ path = IOStreams.path(url, **decode_args(public_send("#{side}_args"), decode: false))
117
+ rescue LoadError
118
+ # A gem that the path needs is not installed in this process, so the path is checked when the job runs.
119
+ nil
120
+ rescue StandardError => e
121
+ # The url is not included in an error message, since it can include credentials.
122
+ if valid_url?(url)
123
+ errors.add(:"#{side}_args", path_error_message(e))
124
+ else
125
+ errors.add(:"#{side}_url", "is not a valid url")
126
+ end
127
+ else
128
+ validate_streams(side, path)
129
+ end
130
+
131
+ def validate_streams(side, path)
132
+ apply_streams(path, public_send("#{side}_streams"), decode: false)
133
+ rescue StandardError => e
134
+ errors.add(:"#{side}_streams", path_error_message(e))
135
+ end
136
+
137
+ # Whether the url on its own is valid, so that a path that is not valid has arguments that are not.
138
+ def valid_url?(url)
139
+ IOStreams.path(url)
140
+ true
141
+ rescue LoadError
142
+ true
143
+ rescue StandardError
144
+ false
145
+ end
146
+
147
+ # The message of an ArgumentError names the argument or stream that is not valid, such as
148
+ # "unknown keyword: :passwrd". Any other error, such as a TypeError from a value of the wrong type, is not
149
+ # shown, since its message can include that value, which can be a secret.
150
+ def path_error_message(exception)
151
+ exception.is_a?(ArgumentError) ? exception.message : "are not valid"
152
+ end
153
+
81
154
  def set_description
82
- self.description ||= "Copying to #{target_url}"
155
+ return if description || target_url.nil?
156
+
157
+ # The url can include credentials, which the display name of the path leaves out.
158
+ self.description = "Copying to #{IOStreams.path(target_url).display_name}"
159
+ rescue StandardError, LoadError
160
+ # The path cannot be created in this process, for example when the gem for its storage is not installed
161
+ # here, or its url is not valid, which #perform reports.
162
+ self.description = "Copying file"
83
163
  end
84
164
 
85
- def apply_streams(path, streams)
86
- streams.each_pair do |stream, args|
87
- stream_args = args.nil? ? {} : decode_args(args)
88
- path.stream(stream.to_sym, **stream_args)
165
+ # Applies the supplied streams to the path, in order. The `none` stream, which `source_streams` and
166
+ # `target_streams` default to, removes the streams before it, so that on its own the file is read or written as
167
+ # it is stored, see `IOStreams::Stream#raw`, rather than with IOStreams' deprecated `stream(:none)`.
168
+ #
169
+ # With `decode: false` the arguments of each stream are not decoded, see #decode_args.
170
+ def apply_streams(path, streams, decode: true)
171
+ streams = streams.to_a
172
+ none = streams.rindex { |stream, _args| stream.to_s == "none" }
173
+ if none
174
+ streams = streams.drop(none + 1)
175
+ return path.raw if streams.empty?
176
+ end
177
+
178
+ streams.each do |stream, args|
179
+ path.stream(stream.to_sym, **decode_args(args, decode: decode))
89
180
  end
90
181
  end
91
182
 
92
- def decode_args(args)
93
- return args.symbolize_keys unless defined?(SymmetricEncryption)
94
-
95
- decoded_args = {}
96
- args.each_pair do |key, value|
97
- if key.to_s.start_with?("encrypted_") && defined?(SymmetricEncryption)
98
- original_key = key.to_s.sub("encrypted_", "").to_sym
99
- decoded_args[original_key] = SymmetricEncryption.decrypt(value)
100
- elsif key.to_s.start_with?("secret_config_") && defined?(SecretConfig)
101
- original_key = key.to_s.sub("secret_config_", "").to_sym
102
- decoded_args[original_key] = SecretConfig.fetch(value)
183
+ # Returns [Hash] the arguments to supply to IOStreams, with the value of each `encrypted_` argument decrypted
184
+ # with Symmetric Encryption, and each `secret_config_` argument fetched from Secret Config, under its name
185
+ # without that prefix. The prefix is kept when its gem is not loaded, so that IOStreams rejects the argument.
186
+ #
187
+ # With `decode: false` the values are as stored, so that the arguments can be checked without decrypting them or
188
+ # fetching any secrets. Each name is without its prefix even when its gem is not loaded, since the job can be
189
+ # created in a process without it, such as an application that only enqueues jobs, or a web interface, and run
190
+ # on a server with it. #perform reports an argument whose gem the server does not load.
191
+ def decode_args(args, decode: true)
192
+ args.to_h do |key, value|
193
+ name = key.to_s
194
+ if name.start_with?("encrypted_") && (!decode || defined?(SymmetricEncryption))
195
+ [name.delete_prefix("encrypted_").to_sym, decode ? SymmetricEncryption.decrypt(value) : value]
196
+ elsif name.start_with?("secret_config_") && (!decode || defined?(SecretConfig))
197
+ [name.delete_prefix("secret_config_").to_sym, decode ? SecretConfig.fetch(value) : value]
103
198
  else
104
- decoded_args[key.to_sym] = value
199
+ [name.to_sym, value]
105
200
  end
106
201
  end
107
- decoded_args
108
202
  end
109
203
  end
110
204
  end
@@ -72,13 +72,16 @@ module RocketJob
72
72
  dirmon_entry.each do |path|
73
73
  # Skip file size checking since S3 files are only visible once completely uploaded.
74
74
  unless path.partial_files_visible?
75
- logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
75
+ logger.info("File: #{dirmon_entry.file_display_name(path)}. Starting: #{dirmon_entry.job_class_name}")
76
76
  dirmon_entry.later(path)
77
77
  next
78
78
  end
79
79
 
80
+ # The name of the file leaves out any credentials, and also the user name, which can select another
81
+ # home directory on the same SFTP server, so the key includes the entry. The name is unique for each
82
+ # file, and valid UTF-8, see DirmonEntry#file_display_name.
80
83
  # BSON Keys cannot contain periods
81
- key = path.to_s.tr(".", "_")
84
+ key = "#{dirmon_entry.id}:#{dirmon_entry.file_display_name(path)}".tr(".", "_")
82
85
  previous_size = previous_file_names[key]
83
86
  # Check every few minutes for a file size change before trying to process the file.
84
87
  size = check_file(dirmon_entry, path, previous_size)
@@ -97,11 +100,11 @@ module RocketJob
97
100
  def check_file(dirmon_entry, path, previous_size)
98
101
  size = path.size
99
102
  if previous_size && (previous_size == size)
100
- logger.info("File stabilized: #{path}. Starting: #{dirmon_entry.job_class_name}")
103
+ logger.info("File stabilized: #{dirmon_entry.file_display_name(path)}. Starting: #{dirmon_entry.job_class_name}")
101
104
  dirmon_entry.later(path)
102
105
  nil
103
106
  else
104
- logger.info("Found file: #{path}. File size: #{size}")
107
+ logger.info("Found file: #{dirmon_entry.file_display_name(path)}. File size: #{size}")
105
108
  # Keep for the next run
106
109
  size
107
110
  end
@@ -11,7 +11,7 @@
11
11
  # CODE
12
12
  # job = RocketJob::Jobs::OnDemandBatchJob.new(code: code, description: 'cleanse users')
13
13
  # arel = User.unscoped.all.order('updated_at DESC')
14
- # job.upload_arel(arel)
14
+ # job.upload(arel)
15
15
  # job.save!
16
16
  #
17
17
  # Console Testing:
@@ -25,7 +25,7 @@
25
25
  #
26
26
  # # Run against a sub-set using a limit
27
27
  # arel = User.unscoped.all.order('updated_at DESC').limit(100)
28
- # job.upload_arel(arel)
28
+ # job.upload(arel)
29
29
  #
30
30
  # # Run the subset directly within the console
31
31
  # job.perform_now
@@ -46,7 +46,7 @@
46
46
  # Example: Move the upload operation into a before_batch.
47
47
  # upload_code = <<-CODE
48
48
  # arel = User.unscoped.all.order('updated_at DESC')
49
- # upload_arel(arel)
49
+ # upload(arel)
50
50
  # CODE
51
51
  #
52
52
  # code = <<-CODE
@@ -108,10 +108,124 @@ module RocketJob
108
108
  output_categories << RocketJob::Category::Output.new(**args)
109
109
  end
110
110
 
111
+ # Matches the code of a job that names a deprecated option, see #schema_proposals, other than an option of upload,
112
+ # see #rocketjob_upload_option_names:
113
+ # - the name of a deprecated category option, see RocketJob::Category::Base.deprecated_option_names, such as
114
+ # `input_category.allowed_columns = %w[a]`,
115
+ # - a deprecated mode, see RocketJob::Category::Input::DEPRECATED_MODES, set as a Symbol or a String, with a Symbol
116
+ # or a String key, such as `mode: :hash`, `mode = "array"`, or `"mode" => "hash"`. A space can only come before
117
+ # `=` or `=>`, so that a ternary, such as `csv ? mode : :hash`, is not matched.
118
+ DEPRECATED_CODE = begin
119
+ category_options = RocketJob::Category::Input.deprecated_option_names |
120
+ RocketJob::Category::Output.deprecated_option_names
121
+ /\b(#{category_options.join('|')})\b|
122
+ \bmode["']?(:|\s*=>?)\s*[:"'](#{RocketJob::Category::Input::DEPRECATED_MODES.join('|')})\b/x
123
+ end
124
+ private_constant :DEPRECATED_CODE
125
+
126
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed for this job's categories, see
127
+ # RocketJob::Batch::Categories#schema_proposals, and a note when its code names a deprecated option, which a
128
+ # proposal cannot translate: a category option or mode, see DEPRECATED_CODE, or an option that it passes to
129
+ # upload, see RocketJob::Batch::IO.deprecated_upload_options. None for a job that has completed or was aborted,
130
+ # since it cannot run again.
131
+ def schema_proposals
132
+ return [] if completed? || aborted?
133
+
134
+ proposals = super
135
+ upload_options = RocketJob::Batch::IO.deprecated_upload_options.map(&:to_s)
136
+ names = %w[code before_code after_code].select do |name|
137
+ source = name == "code" ? perform_source : public_send(name).to_s
138
+ source.match?(DEPRECATED_CODE) || rocketjob_upload_option_names(source).intersect?(upload_options)
139
+ end
140
+ return proposals if names.empty?
141
+
142
+ proposals << RocketJob::SchemaProposal.new(
143
+ where: "#{self.class.name} #{id} (#{state})",
144
+ notes: ["Its #{names.join(', ')} #{names.size == 1 ? 'names' : 'name'} a deprecated category or upload " \
145
+ "option, which needs to be changed by hand."]
146
+ )
147
+ end
148
+
111
149
  private
112
150
 
113
151
  def load_perform_code
114
- instance_eval("def perform(row)\n#{code}\nend", __FILE__, __LINE__)
152
+ instance_eval(perform_source, __FILE__, __LINE__)
153
+ end
154
+
155
+ # Returns [String] the source of #perform, which runs `code`.
156
+ def perform_source
157
+ "def perform(row)\n#{code}\nend"
158
+ end
159
+
160
+ # Returns [Array<String>] the names of the options that the supplied source passes by name in its calls to
161
+ # upload, such as `on_first` for `upload(file, on_first: ...)` or `job.upload file, "delimiter" => "|"`, read with
162
+ # Ruby's own parser, see Ripper. So a variable with the name of an option is not one, nor is an option of another
163
+ # method, such as `IOStreams.path(file_name).each(:line, delimiter: "|")` in the block of upload. Options that
164
+ # the call does not name, such as `upload(file, **options)` or `send(:upload, file, delimiter: "|")`, are not
165
+ # read. None for source that does not parse.
166
+ #
167
+ # The parse tree is walked with a stack of its own, rather than by recursion, so that deeply nested code, such as
168
+ # a long chain of calls, cannot overflow the stack of the thread.
169
+ def rocketjob_upload_option_names(source)
170
+ require "ripper"
171
+
172
+ names = []
173
+ nodes = [Ripper.sexp(source)].grep(Array)
174
+ until nodes.empty?
175
+ node = nodes.pop
176
+ names.concat(rocketjob_option_names(rocketjob_upload_arguments(node)))
177
+ nodes.concat(node.grep(Array))
178
+ end
179
+ names
180
+ end
181
+
182
+ # Returns the node of the arguments of the supplied node of the parse tree when it is a call to upload, with or
183
+ # without parentheses or a receiver, such as `upload(file)`, `upload file` or `job.upload(file)`, otherwise nil.
184
+ def rocketjob_upload_arguments(node)
185
+ case node.first
186
+ when :method_add_arg
187
+ method = node[1].first == :fcall ? node[1][1] : node[1][3] if %i[fcall call].include?(node[1].first)
188
+ node[2][1] if rocketjob_upload?(method) && node[2].first == :arg_paren
189
+ when :command
190
+ node[2] if rocketjob_upload?(node[1])
191
+ when :command_call
192
+ node[4] if rocketjob_upload?(node[3])
193
+ end
194
+ end
195
+
196
+ def rocketjob_upload?(identifier)
197
+ identifier.is_a?(Array) && identifier[0] == :@ident && identifier[1] == "upload"
198
+ end
199
+
200
+ # Returns [Array<String>] the names of the keys of the Hash, with or without braces, among the supplied
201
+ # arguments of a call, but not of a call within them.
202
+ def rocketjob_option_names(arguments)
203
+ return [] unless arguments.is_a?(Array)
204
+
205
+ case arguments.first
206
+ when :bare_assoc_hash
207
+ arguments[1].filter_map { |assoc| rocketjob_key_name(assoc[1]) if assoc.first == :assoc_new }
208
+ when :hash
209
+ rocketjob_option_names([:bare_assoc_hash, arguments.dig(1, 1) || []])
210
+ when :args_add_block, :args_add_star, Array
211
+ arguments.flat_map { |argument| rocketjob_option_names(argument) }
212
+ else
213
+ []
214
+ end
215
+ end
216
+
217
+ # Returns [String] the name of the supplied key of a Hash, such as `on_first:`, `:on_first`, `"on_first" =>` or
218
+ # `"on_first":`, or nil for a key that is not a literal name.
219
+ def rocketjob_key_name(key)
220
+ case key.first
221
+ when :@label
222
+ key[1].delete_suffix(":")
223
+ when :symbol_literal
224
+ key.dig(1, 1, 1)
225
+ when :string_literal, :dyna_symbol
226
+ content = key[1]
227
+ content[1][1] if content.size == 2 && content[1].is_a?(Array) && content[1].first == :@tstring_content
228
+ end
115
229
  end
116
230
 
117
231
  def run_before_code
@@ -18,12 +18,20 @@ module RocketJob
18
18
  # Properties to assign to the job when it is created.
19
19
  field :properties, type: Hash, default: {}, user_editable: true
20
20
 
21
+ # Returns [true] since its properties can hold the deprecated category options of the job that it creates, see
22
+ # #schema_proposals.
23
+ def self.proposes_schemas?
24
+ true
25
+ end
26
+
21
27
  # File to upload
22
28
  field :upload_file_name, type: IOStreams::Path, user_editable: true
23
29
 
24
- # The original Input file name.
30
+ # The original Input file name, without any credentials, see RocketJob::DirmonEntry#file_display_name.
25
31
  # Used by #upload to extract the IOStreams when present.
26
- field :original_file_name, type: String, user_editable: true
32
+ # Shown without any credentials too, see #display_attributes, since a job saved by Rocket Job 7.1, or edited by
33
+ # hand, can hold the url of a file with its user name and password.
34
+ field :original_file_name, type: String, user_editable: true, path: true
27
35
 
28
36
  # Optionally set the job id for the downstream job
29
37
  # Useful when for example the archived file should contain the job id for the downstream job.
@@ -47,6 +55,33 @@ module RocketJob
47
55
  raise(e)
48
56
  end
49
57
 
58
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options in
59
+ # the properties of the job that this job creates, which that job class proposes, see
60
+ # RocketJob::Batch::Categories::ClassMethods#property_schema_proposals. None when its job class is not loaded, or is not a Rocket Job,
61
+ # or when this job has completed or was aborted, since it cannot create its job again. What its job class raises for
62
+ # the properties, such as for a category option that it no longer has, is answered as one proposal with the error,
63
+ # see RocketJob::SchemaProposal.unreadable.
64
+ def schema_proposals
65
+ return [] if completed? || aborted?
66
+
67
+ where = "UploadFileJob #{id} (#{state})"
68
+ klass = rocket_job_class
69
+ return [] unless klass && properties.is_a?(Hash)
70
+
71
+ klass.property_schema_proposals(properties, where)
72
+ rescue StandardError => e
73
+ [RocketJob::SchemaProposal.unreadable(where, e)]
74
+ end
75
+
76
+ # Returns [Hash] the attributes to show, see RocketJob::Plugins::Job::Model#display_attributes, with the paths
77
+ # in the properties shown without any credentials. The job class decides which of its properties are paths.
78
+ def display_attributes
79
+ attrs = super
80
+ klass = rocket_job_class
81
+ attrs["properties"] = klass.display_properties(attrs["properties"]) if klass && attrs["properties"].is_a?(Hash)
82
+ attrs
83
+ end
84
+
50
85
  private
51
86
 
52
87
  def job_class
@@ -85,6 +120,13 @@ module RocketJob
85
120
  end
86
121
  end
87
122
 
123
+ # Returns [Class] the job class when it is defined and is a Rocket Job, otherwise nil, so that it is only asked
124
+ # what a Rocket Job answers, such as #display_attributes.
125
+ def rocket_job_class
126
+ klass = job_class
127
+ klass if klass.is_a?(Class) && klass <= RocketJob::Job
128
+ end
129
+
88
130
  # Validates job_class is a Rocket Job
89
131
  def job_is_a_rocket_job
90
132
  klass = job_class
@@ -106,17 +148,29 @@ module RocketJob
106
148
  end
107
149
 
108
150
  def file_exists
151
+ return upload_file_name_url_valid if upload_file_name.is_a?(String)
109
152
  # Only check for file existence when it is a local file
110
153
  return unless upload_file_name.is_a?(IOStreams::Paths::File)
111
154
  return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
112
155
 
113
156
  return if upload_file_name.exist?
114
157
 
115
- errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
158
+ errors.add(:upload_file_name, "Upload file: #{upload_file_name.display_name} does not exist.")
116
159
  rescue NotImplementedError
117
160
  nil
118
161
  end
119
162
 
163
+ # A url that IOStreams cannot make a path of is read as stored, a String, see IOStreams::Path.demongoize, so one
164
+ # that is set or changed is checked here, rather than when the job runs, see IOStreams::Path.invalid_url?. A saved
165
+ # one is not, so that a job whose url has a scheme that only the servers that run it register can still be saved
166
+ # elsewhere, such as when it is aborted in a web interface.
167
+ def upload_file_name_url_valid
168
+ return unless new_record? || upload_file_name_changed?
169
+ return unless IOStreams::Path.invalid_url?(upload_file_name)
170
+
171
+ errors.add(:upload_file_name, "Upload file: #{RocketJob.path_display_name(upload_file_name)} is not a path or url that IOStreams can read.")
172
+ end
173
+
120
174
  def job_has_properties
121
175
  klass = job_class
122
176
  return unless klass
@@ -13,6 +13,10 @@ module RocketJob
13
13
  class_attribute :user_editable_fields, instance_accessor: false
14
14
  self.user_editable_fields = []
15
15
 
16
+ # Fields that hold a path or url, which can include credentials, see #display_attributes.
17
+ class_attribute :path_fields, instance_accessor: false
18
+ self.path_fields = []
19
+
16
20
  # Attributes to include when copying across the attributes to a new instance on restart.
17
21
  class_attribute :rocket_job_restart_attributes
18
22
  self.rocket_job_restart_attributes = []
@@ -168,6 +172,8 @@ module RocketJob
168
172
  # @option options [ Object, Proc ] :default The field's default
169
173
  # @option options [ Boolean ] :class_attribute Keep the fields default in a class_attribute
170
174
  # @option options [ Boolean ] :user_editable Field can be edited by end users in RJMC
175
+ # @option options [ Boolean ] :path Field holds a path or url, which is shown without its credentials,
176
+ # see #display_attributes. Always true for a field of type IOStreams::Path.
171
177
  #
172
178
  # @return [ Field ] The generated field
173
179
  def field(name, options)
@@ -175,6 +181,9 @@ module RocketJob
175
181
  self.user_editable_fields += [name.to_sym]
176
182
  end
177
183
 
184
+ path = (options.delete(:path) == true) || (options[:type] == IOStreams::Path)
185
+ self.path_fields += [name.to_sym] if path && !path_fields.include?(name.to_sym)
186
+
178
187
  if options.delete(:class_attribute) == true
179
188
  class_attribute(name, instance_accessor: false)
180
189
  public_send("#{name}=", options[:default]) if options.key?(:default)
@@ -193,6 +202,46 @@ module RocketJob
193
202
  def from_properties(properties)
194
203
  new(properties)
195
204
  end
205
+
206
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options
207
+ # that this job class defines, see RocketJob::SchemaProposal. None for a job without categories. Overridden
208
+ # by batch.
209
+ def schema_proposals
210
+ []
211
+ end
212
+
213
+ # Returns [true|false] whether a saved job of this class can hold deprecated category options, whose schema it
214
+ # proposes, see #schema_proposals, so that RocketJob::SchemaProposal.all reads only the saved jobs that can.
215
+ # False for a job without categories. Overridden by batch, and by a job that holds the properties of another
216
+ # job, such as an UploadFileJob.
217
+ def proposes_schemas?
218
+ false
219
+ end
220
+
221
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options in the
222
+ # supplied properties of a job of this class, such as those that a Dirmon entry or an UploadFileJob holds,
223
+ # at the supplied place. None for a job without categories. Overridden by batch.
224
+ def property_schema_proposals(_properties, _where)
225
+ []
226
+ end
227
+
228
+ # Returns [Hash] the supplied attributes or properties of this job class to show to people, with the value
229
+ # of each field that holds a path or url shown by its display name, without any credentials,
230
+ # see `path_fields`. Overridden by batch to also show the paths in its categories.
231
+ #
232
+ # For example, the properties that a Dirmon entry or an UploadFileJob holds for this job class.
233
+ def display_properties(properties)
234
+ properties.to_h do |key, value|
235
+ [key, path_fields.include?(key.to_sym) ? RocketJob.path_display_name(value) : value]
236
+ end
237
+ end
238
+ end
239
+
240
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options
241
+ # that this saved job holds, see RocketJob::SchemaProposal. None for a job without categories. Overridden by
242
+ # batch, and by a job that holds the properties of another job, such as an UploadFileJob.
243
+ def schema_proposals
244
+ []
196
245
  end
197
246
 
198
247
  # Returns [Float] the number of seconds the job has taken
@@ -254,6 +303,12 @@ module RocketJob
254
303
  run_at || created_at
255
304
  end
256
305
 
306
+ # Returns [Hash] the attributes of this job to show to people, for example in Rocket Job Mission Control,
307
+ # with each path or url shown by its display name, without any credentials, see `.display_properties`.
308
+ def display_attributes
309
+ self.class.display_properties(attributes)
310
+ end
311
+
257
312
  # Returns [Hash] status of this job
258
313
  def as_json
259
314
  attrs = serializable_hash(methods: %i[seconds duration])
@@ -27,41 +27,42 @@ module RocketJob
27
27
  # MyCronJob.create!(retry_limit: 0)
28
28
  #
29
29
  # Below is a table of the delay for each retry attempt, as well as the _total_ duration spent retrying a job for
30
- # the specified number of retries, excluding actual processing time.
30
+ # the specified number of retries, excluding actual processing time. The delay before retry `n` is
31
+ # `(n - 1)**4 + 5` seconds.
31
32
  #
32
33
  # |---------|------------|----------------|
33
34
  # | Attempt | Delay | Total Duration |
34
35
  # |---------|------------|----------------|
35
- # | 01 | 6.000s | 6.000s |
36
- # | 02 | 21.000s | 27.000s |
37
- # | 03 | 1m 26s | 1m 53s |
38
- # | 04 | 4m 21s | 6m 14s |
39
- # | 05 | 10m 30s | 16m 44s |
40
- # | 06 | 21m 41s | 38m 25s |
41
- # | 07 | 40m 6s | 20h 18m |
42
- # | 08 | 20h 8m | 21h 26m |
43
- # | 09 | 20h 49m | 23h 16m |
44
- # | 10 | 21h 46m | 2h 3m |
45
- # | 11 | 23h 4m | 6h 7m |
46
- # | 12 | 0h 45m | 11h 52m |
47
- # | 13 | 2h 56m | 1d 19h 48m |
48
- # | 14 | 5h 40m | 1d 6h 29m |
49
- # | 15 | 9h 3m | 2d 20h 33m |
50
- # | 16 | 13h 12m | 2d 14h 45m |
51
- # | 17 | 18h 12m | 3d 13h 57m |
52
- # | 18 | 1d 0h 9m | 5d 19h 7m |
53
- # | 19 | 1d 7h 12m | 6d 7h 19m |
54
- # | 20 | 1d 15h 26m | 8d 3h 46m |
55
- # | 21 | 2d 1h 1m | 10d 9h 47m |
56
- # | 22 | 2d 12h 4m | 13d 2h 51m |
57
- # | 23 | 3d 0h 44m | 16d 8h 35m |
58
- # | 24 | 3d 15h 9m | 20d 4h 45m |
59
- # | 25 | 4d 7h 30m | 24d 17h 16 |
60
- # | 26 | 5d 1h 56m | 30d 0h 12m |
61
- # | 27 | 6d 22h 37m | 36d 3h 49m |
62
- # | 28 | 7d 21h 44m | 43d 6h 34m |
63
- # | 29 | 8d 23h 28m | 51d 11h 2m |
64
- # | 30 | 9d 4h 0m | 61d 20h 2m |
36
+ # | 01 | 5.000s | 5.000s |
37
+ # | 02 | 6.000s | 11.000s |
38
+ # | 03 | 21.000s | 32.000s |
39
+ # | 04 | 1m 26s | 1m 58s |
40
+ # | 05 | 4m 21s | 6m 19s |
41
+ # | 06 | 10m 30s | 16m 49s |
42
+ # | 07 | 21m 41s | 38m 30s |
43
+ # | 08 | 40m 6s | 1h 18m |
44
+ # | 09 | 1h 8m | 2h 26m |
45
+ # | 10 | 1h 49m | 4h 16m |
46
+ # | 11 | 2h 46m | 7h 3m |
47
+ # | 12 | 4h 4m | 11h 7m |
48
+ # | 13 | 5h 45m | 16h 52m |
49
+ # | 14 | 7h 56m | 1d 0h 49m |
50
+ # | 15 | 10h 40m | 1d 11h 29m |
51
+ # | 16 | 14h 3m | 2d 1h 33m |
52
+ # | 17 | 18h 12m | 2d 19h 45m |
53
+ # | 18 | 23h 12m | 3d 18h 57m |
54
+ # | 19 | 1d 5h 9m | 5d 0h 7m |
55
+ # | 20 | 1d 12h 12m | 6d 12h 19m |
56
+ # | 21 | 1d 20h 26m | 8d 8h 46m |
57
+ # | 22 | 2d 6h 1m | 10d 14h 47m |
58
+ # | 23 | 2d 17h 4m | 13d 7h 51m |
59
+ # | 24 | 3d 5h 44m | 16d 13h 36m |
60
+ # | 25 | 3d 20h 9m | 20d 9h 45m |
61
+ # | 26 | 4d 12h 30m | 24d 22h 16m |
62
+ # | 27 | 5d 6h 56m | 30d 5h 12m |
63
+ # | 28 | 6d 3h 37m | 36d 8h 50m |
64
+ # | 29 | 7d 2h 44m | 43d 11h 34m |
65
+ # | 30 | 8d 4h 28m | 51d 16h 2m |
65
66
  # |---------|------------|----------------|
66
67
  module Retry
67
68
  extend ActiveSupport::Concern
@@ -70,7 +71,7 @@ module RocketJob
70
71
  after_fail :rocket_job_retry
71
72
 
72
73
  # Maximum number of times to retry this job.
73
- # The default of 25 is almost 25 days of retries.
74
+ # The default of 25 retries spans about 20 days, see the table above.
74
75
  field :retry_limit, type: Integer, default: 25, class_attribute: true, user_editable: true, copy_on_restart: true
75
76
 
76
77
  # List of times when this job failed
@@ -115,7 +116,7 @@ module RocketJob
115
116
  #
116
117
  # For example, to see the durations for the first 25 retries:
117
118
  # count = 25
118
- # intervals = (1..count).map { |attempt| attempt**4 + 5 }
119
+ # intervals = (0...count).map { |failures| (failures**4) + 5 }
119
120
  #
120
121
  # Display the above intervals as human readable durations:
121
122
  # intervals.map { |seconds| RocketJob.seconds_as_duration(seconds) }
@@ -124,7 +125,7 @@ module RocketJob
124
125
  # RocketJob.seconds_as_duration(intervals.sum)
125
126
  #
126
127
  # Or, to see the total durations based on the number of retries:
127
- # (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
128
+ # (0...count).map { |i| "#{i + 1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}" }
128
129
  def rocket_job_retry_seconds_to_delay
129
130
  (rocket_job_failure_count**4) + 5
130
131
  end