rocketjob 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rocket_job/batch/categories.rb +145 -37
- data/lib/rocket_job/batch/io.rb +222 -141
- data/lib/rocket_job/category/base.rb +57 -0
- data/lib/rocket_job/category/input.rb +252 -62
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +32 -0
- data/lib/rocket_job/deprecation.rb +12 -2
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +103 -19
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +9 -2
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +4 -0
- metadata +6 -2
|
@@ -29,8 +29,8 @@ module RocketJob
|
|
|
29
29
|
self.priority = 30
|
|
30
30
|
|
|
31
31
|
# File names in IOStreams URL format.
|
|
32
|
-
field :source_url, type: String, user_editable: true
|
|
33
|
-
field :target_url, type: String, user_editable: true
|
|
32
|
+
field :source_url, type: String, user_editable: true, path: true
|
|
33
|
+
field :target_url, type: String, user_editable: true, path: true
|
|
34
34
|
|
|
35
35
|
# Any optional arguments to pass through to the IOStreams source and/or target.
|
|
36
36
|
field :source_args, type: Hash, default: -> { {} }, user_editable: true
|
|
@@ -51,6 +51,8 @@ module RocketJob
|
|
|
51
51
|
validates_presence_of :source_url, unless: :source_data
|
|
52
52
|
validates_presence_of :target_url
|
|
53
53
|
validates_presence_of :source_data, unless: :source_url
|
|
54
|
+
validate :source_path_is_valid, if: -> { source_url && path_changed?(:source) }
|
|
55
|
+
validate :target_path_is_valid, if: -> { target_url && path_changed?(:target) }
|
|
54
56
|
|
|
55
57
|
before_save :set_description
|
|
56
58
|
|
|
@@ -76,16 +78,96 @@ module RocketJob
|
|
|
76
78
|
target
|
|
77
79
|
end
|
|
78
80
|
|
|
81
|
+
# Returns [Hash] the attributes to show, see RocketJob::Plugins::Job::Model#display_attributes, with the secrets
|
|
82
|
+
# in the arguments and streams of the source and target replaced, such as an SFTP password or a PGP passphrase.
|
|
83
|
+
# IOStreams decides which are secret, from the kind of path of each url.
|
|
84
|
+
def display_attributes
|
|
85
|
+
attrs = super
|
|
86
|
+
%w[source target].each do |side|
|
|
87
|
+
args = attrs["#{side}_args"]
|
|
88
|
+
streams = attrs["#{side}_streams"]
|
|
89
|
+
attrs["#{side}_args"] = IOStreams.redact_path_options(self["#{side}_url"].to_s, args) if args.is_a?(Hash)
|
|
90
|
+
attrs["#{side}_streams"] = IOStreams.redact_stream_options(streams) if streams.is_a?(Hash)
|
|
91
|
+
end
|
|
92
|
+
attrs
|
|
93
|
+
end
|
|
94
|
+
|
|
79
95
|
private
|
|
80
96
|
|
|
97
|
+
def source_path_is_valid
|
|
98
|
+
validate_path(:source)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def target_path_is_valid
|
|
102
|
+
validate_path(:target)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Whether the url, arguments or streams of the source or target are new or changed, so that a job whose path
|
|
106
|
+
# was valid when it was created, can still be saved, for example when it fails.
|
|
107
|
+
def path_changed?(side)
|
|
108
|
+
new_record? || %w[url args streams].any? { |name| attribute_changed?("#{side}_#{name}") }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Builds the path of the source or target without using it, so that IOStreams checks its url, the names and
|
|
112
|
+
# values of its arguments, and its streams, in the same way as #perform. Encrypted arguments are not decrypted,
|
|
113
|
+
# nor secrets fetched, since their values are not used.
|
|
114
|
+
def validate_path(side)
|
|
115
|
+
url = public_send("#{side}_url")
|
|
116
|
+
path = IOStreams.path(url, **decode_args(public_send("#{side}_args"), decode: false))
|
|
117
|
+
rescue LoadError
|
|
118
|
+
# A gem that the path needs is not installed in this process, so the path is checked when the job runs.
|
|
119
|
+
nil
|
|
120
|
+
rescue StandardError => e
|
|
121
|
+
# The url is not included in an error message, since it can include credentials.
|
|
122
|
+
if valid_url?(url)
|
|
123
|
+
errors.add(:"#{side}_args", path_error_message(e))
|
|
124
|
+
else
|
|
125
|
+
errors.add(:"#{side}_url", "is not a valid url")
|
|
126
|
+
end
|
|
127
|
+
else
|
|
128
|
+
validate_streams(side, path)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def validate_streams(side, path)
|
|
132
|
+
apply_streams(path, public_send("#{side}_streams"), decode: false)
|
|
133
|
+
rescue StandardError => e
|
|
134
|
+
errors.add(:"#{side}_streams", path_error_message(e))
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Whether the url on its own is valid, so that a path that is not valid has arguments that are not.
|
|
138
|
+
def valid_url?(url)
|
|
139
|
+
IOStreams.path(url)
|
|
140
|
+
true
|
|
141
|
+
rescue LoadError
|
|
142
|
+
true
|
|
143
|
+
rescue StandardError
|
|
144
|
+
false
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# The message of an ArgumentError names the argument or stream that is not valid, such as
|
|
148
|
+
# "unknown keyword: :passwrd". Any other error, such as a TypeError from a value of the wrong type, is not
|
|
149
|
+
# shown, since its message can include that value, which can be a secret.
|
|
150
|
+
def path_error_message(exception)
|
|
151
|
+
exception.is_a?(ArgumentError) ? exception.message : "are not valid"
|
|
152
|
+
end
|
|
153
|
+
|
|
81
154
|
def set_description
|
|
82
|
-
|
|
155
|
+
return if description || target_url.nil?
|
|
156
|
+
|
|
157
|
+
# The url can include credentials, which the display name of the path leaves out.
|
|
158
|
+
self.description = "Copying to #{IOStreams.path(target_url).display_name}"
|
|
159
|
+
rescue StandardError, LoadError
|
|
160
|
+
# The path cannot be created in this process, for example when the gem for its storage is not installed
|
|
161
|
+
# here, or its url is not valid, which #perform reports.
|
|
162
|
+
self.description = "Copying file"
|
|
83
163
|
end
|
|
84
164
|
|
|
85
165
|
# Applies the supplied streams to the path, in order. The `none` stream, which `source_streams` and
|
|
86
166
|
# `target_streams` default to, removes the streams before it, so that on its own the file is read or written as
|
|
87
167
|
# it is stored, see `IOStreams::Stream#raw`, rather than with IOStreams' deprecated `stream(:none)`.
|
|
88
|
-
|
|
168
|
+
#
|
|
169
|
+
# With `decode: false` the arguments of each stream are not decoded, see #decode_args.
|
|
170
|
+
def apply_streams(path, streams, decode: true)
|
|
89
171
|
streams = streams.to_a
|
|
90
172
|
none = streams.rindex { |stream, _args| stream.to_s == "none" }
|
|
91
173
|
if none
|
|
@@ -94,27 +176,29 @@ module RocketJob
|
|
|
94
176
|
end
|
|
95
177
|
|
|
96
178
|
streams.each do |stream, args|
|
|
97
|
-
|
|
98
|
-
path.stream(stream.to_sym, **stream_args)
|
|
179
|
+
path.stream(stream.to_sym, **decode_args(args, decode: decode))
|
|
99
180
|
end
|
|
100
181
|
end
|
|
101
182
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
183
|
+
# Returns [Hash] the arguments to supply to IOStreams, with the value of each `encrypted_` argument decrypted
|
|
184
|
+
# with Symmetric Encryption, and each `secret_config_` argument fetched from Secret Config, under its name
|
|
185
|
+
# without that prefix. The prefix is kept when its gem is not loaded, so that IOStreams rejects the argument.
|
|
186
|
+
#
|
|
187
|
+
# With `decode: false` the values are as stored, so that the arguments can be checked without decrypting them or
|
|
188
|
+
# fetching any secrets. Each name is without its prefix even when its gem is not loaded, since the job can be
|
|
189
|
+
# created in a process without it, such as an application that only enqueues jobs, or a web interface, and run
|
|
190
|
+
# on a server with it. #perform reports an argument whose gem the server does not load.
|
|
191
|
+
def decode_args(args, decode: true)
|
|
192
|
+
args.to_h do |key, value|
|
|
193
|
+
name = key.to_s
|
|
194
|
+
if name.start_with?("encrypted_") && (!decode || defined?(SymmetricEncryption))
|
|
195
|
+
[name.delete_prefix("encrypted_").to_sym, decode ? SymmetricEncryption.decrypt(value) : value]
|
|
196
|
+
elsif name.start_with?("secret_config_") && (!decode || defined?(SecretConfig))
|
|
197
|
+
[name.delete_prefix("secret_config_").to_sym, decode ? SecretConfig.fetch(value) : value]
|
|
113
198
|
else
|
|
114
|
-
|
|
199
|
+
[name.to_sym, value]
|
|
115
200
|
end
|
|
116
201
|
end
|
|
117
|
-
decoded_args
|
|
118
202
|
end
|
|
119
203
|
end
|
|
120
204
|
end
|
|
@@ -72,13 +72,16 @@ module RocketJob
|
|
|
72
72
|
dirmon_entry.each do |path|
|
|
73
73
|
# Skip file size checking since S3 files are only visible once completely uploaded.
|
|
74
74
|
unless path.partial_files_visible?
|
|
75
|
-
logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
|
|
75
|
+
logger.info("File: #{dirmon_entry.file_display_name(path)}. Starting: #{dirmon_entry.job_class_name}")
|
|
76
76
|
dirmon_entry.later(path)
|
|
77
77
|
next
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
# The name of the file leaves out any credentials, and also the user name, which can select another
|
|
81
|
+
# home directory on the same SFTP server, so the key includes the entry. The name is unique for each
|
|
82
|
+
# file, and valid UTF-8, see DirmonEntry#file_display_name.
|
|
80
83
|
# BSON Keys cannot contain periods
|
|
81
|
-
key = path.
|
|
84
|
+
key = "#{dirmon_entry.id}:#{dirmon_entry.file_display_name(path)}".tr(".", "_")
|
|
82
85
|
previous_size = previous_file_names[key]
|
|
83
86
|
# Check every few minutes for a file size change before trying to process the file.
|
|
84
87
|
size = check_file(dirmon_entry, path, previous_size)
|
|
@@ -97,11 +100,11 @@ module RocketJob
|
|
|
97
100
|
def check_file(dirmon_entry, path, previous_size)
|
|
98
101
|
size = path.size
|
|
99
102
|
if previous_size && (previous_size == size)
|
|
100
|
-
logger.info("File stabilized: #{path}. Starting: #{dirmon_entry.job_class_name}")
|
|
103
|
+
logger.info("File stabilized: #{dirmon_entry.file_display_name(path)}. Starting: #{dirmon_entry.job_class_name}")
|
|
101
104
|
dirmon_entry.later(path)
|
|
102
105
|
nil
|
|
103
106
|
else
|
|
104
|
-
logger.info("Found file: #{path}. File size: #{size}")
|
|
107
|
+
logger.info("Found file: #{dirmon_entry.file_display_name(path)}. File size: #{size}")
|
|
105
108
|
# Keep for the next run
|
|
106
109
|
size
|
|
107
110
|
end
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# CODE
|
|
12
12
|
# job = RocketJob::Jobs::OnDemandBatchJob.new(code: code, description: 'cleanse users')
|
|
13
13
|
# arel = User.unscoped.all.order('updated_at DESC')
|
|
14
|
-
# job.
|
|
14
|
+
# job.upload(arel)
|
|
15
15
|
# job.save!
|
|
16
16
|
#
|
|
17
17
|
# Console Testing:
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
#
|
|
26
26
|
# # Run against a sub-set using a limit
|
|
27
27
|
# arel = User.unscoped.all.order('updated_at DESC').limit(100)
|
|
28
|
-
# job.
|
|
28
|
+
# job.upload(arel)
|
|
29
29
|
#
|
|
30
30
|
# # Run the subset directly within the console
|
|
31
31
|
# job.perform_now
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
# Example: Move the upload operation into a before_batch.
|
|
47
47
|
# upload_code = <<-CODE
|
|
48
48
|
# arel = User.unscoped.all.order('updated_at DESC')
|
|
49
|
-
#
|
|
49
|
+
# upload(arel)
|
|
50
50
|
# CODE
|
|
51
51
|
#
|
|
52
52
|
# code = <<-CODE
|
|
@@ -108,10 +108,124 @@ module RocketJob
|
|
|
108
108
|
output_categories << RocketJob::Category::Output.new(**args)
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
+
# Matches the code of a job that names a deprecated option, see #schema_proposals, other than an option of upload,
|
|
112
|
+
# see #rocketjob_upload_option_names:
|
|
113
|
+
# - the name of a deprecated category option, see RocketJob::Category::Base.deprecated_option_names, such as
|
|
114
|
+
# `input_category.allowed_columns = %w[a]`,
|
|
115
|
+
# - a deprecated mode, see RocketJob::Category::Input::DEPRECATED_MODES, set as a Symbol or a String, with a Symbol
|
|
116
|
+
# or a String key, such as `mode: :hash`, `mode = "array"`, or `"mode" => "hash"`. A space can only come before
|
|
117
|
+
# `=` or `=>`, so that a ternary, such as `csv ? mode : :hash`, is not matched.
|
|
118
|
+
DEPRECATED_CODE = begin
|
|
119
|
+
category_options = RocketJob::Category::Input.deprecated_option_names |
|
|
120
|
+
RocketJob::Category::Output.deprecated_option_names
|
|
121
|
+
/\b(#{category_options.join('|')})\b|
|
|
122
|
+
\bmode["']?(:|\s*=>?)\s*[:"'](#{RocketJob::Category::Input::DEPRECATED_MODES.join('|')})\b/x
|
|
123
|
+
end
|
|
124
|
+
private_constant :DEPRECATED_CODE
|
|
125
|
+
|
|
126
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed for this job's categories, see
|
|
127
|
+
# RocketJob::Batch::Categories#schema_proposals, and a note when its code names a deprecated option, which a
|
|
128
|
+
# proposal cannot translate: a category option or mode, see DEPRECATED_CODE, or an option that it passes to
|
|
129
|
+
# upload, see RocketJob::Batch::IO.deprecated_upload_options. None for a job that has completed or was aborted,
|
|
130
|
+
# since it cannot run again.
|
|
131
|
+
def schema_proposals
|
|
132
|
+
return [] if completed? || aborted?
|
|
133
|
+
|
|
134
|
+
proposals = super
|
|
135
|
+
upload_options = RocketJob::Batch::IO.deprecated_upload_options.map(&:to_s)
|
|
136
|
+
names = %w[code before_code after_code].select do |name|
|
|
137
|
+
source = name == "code" ? perform_source : public_send(name).to_s
|
|
138
|
+
source.match?(DEPRECATED_CODE) || rocketjob_upload_option_names(source).intersect?(upload_options)
|
|
139
|
+
end
|
|
140
|
+
return proposals if names.empty?
|
|
141
|
+
|
|
142
|
+
proposals << RocketJob::SchemaProposal.new(
|
|
143
|
+
where: "#{self.class.name} #{id} (#{state})",
|
|
144
|
+
notes: ["Its #{names.join(', ')} #{names.size == 1 ? 'names' : 'name'} a deprecated category or upload " \
|
|
145
|
+
"option, which needs to be changed by hand."]
|
|
146
|
+
)
|
|
147
|
+
end
|
|
148
|
+
|
|
111
149
|
private
|
|
112
150
|
|
|
113
151
|
def load_perform_code
|
|
114
|
-
instance_eval(
|
|
152
|
+
instance_eval(perform_source, __FILE__, __LINE__)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Returns [String] the source of #perform, which runs `code`.
|
|
156
|
+
def perform_source
|
|
157
|
+
"def perform(row)\n#{code}\nend"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Returns [Array<String>] the names of the options that the supplied source passes by name in its calls to
|
|
161
|
+
# upload, such as `on_first` for `upload(file, on_first: ...)` or `job.upload file, "delimiter" => "|"`, read with
|
|
162
|
+
# Ruby's own parser, see Ripper. So a variable with the name of an option is not one, nor is an option of another
|
|
163
|
+
# method, such as `IOStreams.path(file_name).each(:line, delimiter: "|")` in the block of upload. Options that
|
|
164
|
+
# the call does not name, such as `upload(file, **options)` or `send(:upload, file, delimiter: "|")`, are not
|
|
165
|
+
# read. None for source that does not parse.
|
|
166
|
+
#
|
|
167
|
+
# The parse tree is walked with a stack of its own, rather than by recursion, so that deeply nested code, such as
|
|
168
|
+
# a long chain of calls, cannot overflow the stack of the thread.
|
|
169
|
+
def rocketjob_upload_option_names(source)
|
|
170
|
+
require "ripper"
|
|
171
|
+
|
|
172
|
+
names = []
|
|
173
|
+
nodes = [Ripper.sexp(source)].grep(Array)
|
|
174
|
+
until nodes.empty?
|
|
175
|
+
node = nodes.pop
|
|
176
|
+
names.concat(rocketjob_option_names(rocketjob_upload_arguments(node)))
|
|
177
|
+
nodes.concat(node.grep(Array))
|
|
178
|
+
end
|
|
179
|
+
names
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Returns the node of the arguments of the supplied node of the parse tree when it is a call to upload, with or
|
|
183
|
+
# without parentheses or a receiver, such as `upload(file)`, `upload file` or `job.upload(file)`, otherwise nil.
|
|
184
|
+
def rocketjob_upload_arguments(node)
|
|
185
|
+
case node.first
|
|
186
|
+
when :method_add_arg
|
|
187
|
+
method = node[1].first == :fcall ? node[1][1] : node[1][3] if %i[fcall call].include?(node[1].first)
|
|
188
|
+
node[2][1] if rocketjob_upload?(method) && node[2].first == :arg_paren
|
|
189
|
+
when :command
|
|
190
|
+
node[2] if rocketjob_upload?(node[1])
|
|
191
|
+
when :command_call
|
|
192
|
+
node[4] if rocketjob_upload?(node[3])
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def rocketjob_upload?(identifier)
|
|
197
|
+
identifier.is_a?(Array) && identifier[0] == :@ident && identifier[1] == "upload"
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Returns [Array<String>] the names of the keys of the Hash, with or without braces, among the supplied
|
|
201
|
+
# arguments of a call, but not of a call within them.
|
|
202
|
+
def rocketjob_option_names(arguments)
|
|
203
|
+
return [] unless arguments.is_a?(Array)
|
|
204
|
+
|
|
205
|
+
case arguments.first
|
|
206
|
+
when :bare_assoc_hash
|
|
207
|
+
arguments[1].filter_map { |assoc| rocketjob_key_name(assoc[1]) if assoc.first == :assoc_new }
|
|
208
|
+
when :hash
|
|
209
|
+
rocketjob_option_names([:bare_assoc_hash, arguments.dig(1, 1) || []])
|
|
210
|
+
when :args_add_block, :args_add_star, Array
|
|
211
|
+
arguments.flat_map { |argument| rocketjob_option_names(argument) }
|
|
212
|
+
else
|
|
213
|
+
[]
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# Returns [String] the name of the supplied key of a Hash, such as `on_first:`, `:on_first`, `"on_first" =>` or
|
|
218
|
+
# `"on_first":`, or nil for a key that is not a literal name.
|
|
219
|
+
def rocketjob_key_name(key)
|
|
220
|
+
case key.first
|
|
221
|
+
when :@label
|
|
222
|
+
key[1].delete_suffix(":")
|
|
223
|
+
when :symbol_literal
|
|
224
|
+
key.dig(1, 1, 1)
|
|
225
|
+
when :string_literal, :dyna_symbol
|
|
226
|
+
content = key[1]
|
|
227
|
+
content[1][1] if content.size == 2 && content[1].is_a?(Array) && content[1].first == :@tstring_content
|
|
228
|
+
end
|
|
115
229
|
end
|
|
116
230
|
|
|
117
231
|
def run_before_code
|
|
@@ -18,12 +18,20 @@ module RocketJob
|
|
|
18
18
|
# Properties to assign to the job when it is created.
|
|
19
19
|
field :properties, type: Hash, default: {}, user_editable: true
|
|
20
20
|
|
|
21
|
+
# Returns [true] since its properties can hold the deprecated category options of the job that it creates, see
|
|
22
|
+
# #schema_proposals.
|
|
23
|
+
def self.proposes_schemas?
|
|
24
|
+
true
|
|
25
|
+
end
|
|
26
|
+
|
|
21
27
|
# File to upload
|
|
22
28
|
field :upload_file_name, type: IOStreams::Path, user_editable: true
|
|
23
29
|
|
|
24
|
-
# The original Input file name.
|
|
30
|
+
# The original Input file name, without any credentials, see RocketJob::DirmonEntry#file_display_name.
|
|
25
31
|
# Used by #upload to extract the IOStreams when present.
|
|
26
|
-
|
|
32
|
+
# Shown without any credentials too, see #display_attributes, since a job saved by Rocket Job 7.1, or edited by
|
|
33
|
+
# hand, can hold the url of a file with its user name and password.
|
|
34
|
+
field :original_file_name, type: String, user_editable: true, path: true
|
|
27
35
|
|
|
28
36
|
# Optionally set the job id for the downstream job
|
|
29
37
|
# Useful when for example the archived file should contain the job id for the downstream job.
|
|
@@ -47,6 +55,33 @@ module RocketJob
|
|
|
47
55
|
raise(e)
|
|
48
56
|
end
|
|
49
57
|
|
|
58
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options in
|
|
59
|
+
# the properties of the job that this job creates, which that job class proposes, see
|
|
60
|
+
# RocketJob::Batch::Categories::ClassMethods#property_schema_proposals. None when its job class is not loaded, or is not a Rocket Job,
|
|
61
|
+
# or when this job has completed or was aborted, since it cannot create its job again. What its job class raises for
|
|
62
|
+
# the properties, such as for a category option that it no longer has, is answered as one proposal with the error,
|
|
63
|
+
# see RocketJob::SchemaProposal.unreadable.
|
|
64
|
+
def schema_proposals
|
|
65
|
+
return [] if completed? || aborted?
|
|
66
|
+
|
|
67
|
+
where = "UploadFileJob #{id} (#{state})"
|
|
68
|
+
klass = rocket_job_class
|
|
69
|
+
return [] unless klass && properties.is_a?(Hash)
|
|
70
|
+
|
|
71
|
+
klass.property_schema_proposals(properties, where)
|
|
72
|
+
rescue StandardError => e
|
|
73
|
+
[RocketJob::SchemaProposal.unreadable(where, e)]
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Returns [Hash] the attributes to show, see RocketJob::Plugins::Job::Model#display_attributes, with the paths
|
|
77
|
+
# in the properties shown without any credentials. The job class decides which of its properties are paths.
|
|
78
|
+
def display_attributes
|
|
79
|
+
attrs = super
|
|
80
|
+
klass = rocket_job_class
|
|
81
|
+
attrs["properties"] = klass.display_properties(attrs["properties"]) if klass && attrs["properties"].is_a?(Hash)
|
|
82
|
+
attrs
|
|
83
|
+
end
|
|
84
|
+
|
|
50
85
|
private
|
|
51
86
|
|
|
52
87
|
def job_class
|
|
@@ -85,6 +120,13 @@ module RocketJob
|
|
|
85
120
|
end
|
|
86
121
|
end
|
|
87
122
|
|
|
123
|
+
# Returns [Class] the job class when it is defined and is a Rocket Job, otherwise nil, so that it is only asked
|
|
124
|
+
# what a Rocket Job answers, such as #display_attributes.
|
|
125
|
+
def rocket_job_class
|
|
126
|
+
klass = job_class
|
|
127
|
+
klass if klass.is_a?(Class) && klass <= RocketJob::Job
|
|
128
|
+
end
|
|
129
|
+
|
|
88
130
|
# Validates job_class is a Rocket Job
|
|
89
131
|
def job_is_a_rocket_job
|
|
90
132
|
klass = job_class
|
|
@@ -106,17 +148,29 @@ module RocketJob
|
|
|
106
148
|
end
|
|
107
149
|
|
|
108
150
|
def file_exists
|
|
151
|
+
return upload_file_name_url_valid if upload_file_name.is_a?(String)
|
|
109
152
|
# Only check for file existence when it is a local file
|
|
110
153
|
return unless upload_file_name.is_a?(IOStreams::Paths::File)
|
|
111
154
|
return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
|
|
112
155
|
|
|
113
156
|
return if upload_file_name.exist?
|
|
114
157
|
|
|
115
|
-
errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
|
|
158
|
+
errors.add(:upload_file_name, "Upload file: #{upload_file_name.display_name} does not exist.")
|
|
116
159
|
rescue NotImplementedError
|
|
117
160
|
nil
|
|
118
161
|
end
|
|
119
162
|
|
|
163
|
+
# A url that IOStreams cannot make a path of is read as stored, a String, see IOStreams::Path.demongoize, so one
|
|
164
|
+
# that is set or changed is checked here, rather than when the job runs, see IOStreams::Path.invalid_url?. A saved
|
|
165
|
+
# one is not, so that a job whose url has a scheme that only the servers that run it register can still be saved
|
|
166
|
+
# elsewhere, such as when it is aborted in a web interface.
|
|
167
|
+
def upload_file_name_url_valid
|
|
168
|
+
return unless new_record? || upload_file_name_changed?
|
|
169
|
+
return unless IOStreams::Path.invalid_url?(upload_file_name)
|
|
170
|
+
|
|
171
|
+
errors.add(:upload_file_name, "Upload file: #{RocketJob.path_display_name(upload_file_name)} is not a path or url that IOStreams can read.")
|
|
172
|
+
end
|
|
173
|
+
|
|
120
174
|
def job_has_properties
|
|
121
175
|
klass = job_class
|
|
122
176
|
return unless klass
|
|
@@ -13,6 +13,10 @@ module RocketJob
|
|
|
13
13
|
class_attribute :user_editable_fields, instance_accessor: false
|
|
14
14
|
self.user_editable_fields = []
|
|
15
15
|
|
|
16
|
+
# Fields that hold a path or url, which can include credentials, see #display_attributes.
|
|
17
|
+
class_attribute :path_fields, instance_accessor: false
|
|
18
|
+
self.path_fields = []
|
|
19
|
+
|
|
16
20
|
# Attributes to include when copying across the attributes to a new instance on restart.
|
|
17
21
|
class_attribute :rocket_job_restart_attributes
|
|
18
22
|
self.rocket_job_restart_attributes = []
|
|
@@ -168,6 +172,8 @@ module RocketJob
|
|
|
168
172
|
# @option options [ Object, Proc ] :default The field's default
|
|
169
173
|
# @option options [ Boolean ] :class_attribute Keep the fields default in a class_attribute
|
|
170
174
|
# @option options [ Boolean ] :user_editable Field can be edited by end users in RJMC
|
|
175
|
+
# @option options [ Boolean ] :path Field holds a path or url, which is shown without its credentials,
|
|
176
|
+
# see #display_attributes. Always true for a field of type IOStreams::Path.
|
|
171
177
|
#
|
|
172
178
|
# @return [ Field ] The generated field
|
|
173
179
|
def field(name, options)
|
|
@@ -175,6 +181,9 @@ module RocketJob
|
|
|
175
181
|
self.user_editable_fields += [name.to_sym]
|
|
176
182
|
end
|
|
177
183
|
|
|
184
|
+
path = (options.delete(:path) == true) || (options[:type] == IOStreams::Path)
|
|
185
|
+
self.path_fields += [name.to_sym] if path && !path_fields.include?(name.to_sym)
|
|
186
|
+
|
|
178
187
|
if options.delete(:class_attribute) == true
|
|
179
188
|
class_attribute(name, instance_accessor: false)
|
|
180
189
|
public_send("#{name}=", options[:default]) if options.key?(:default)
|
|
@@ -193,6 +202,46 @@ module RocketJob
|
|
|
193
202
|
def from_properties(properties)
|
|
194
203
|
new(properties)
|
|
195
204
|
end
|
|
205
|
+
|
|
206
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options
|
|
207
|
+
# that this job class defines, see RocketJob::SchemaProposal. None for a job without categories. Overridden
|
|
208
|
+
# by batch.
|
|
209
|
+
def schema_proposals
|
|
210
|
+
[]
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Returns [true|false] whether a saved job of this class can hold deprecated category options, whose schema it
|
|
214
|
+
# proposes, see #schema_proposals, so that RocketJob::SchemaProposal.all reads only the saved jobs that can.
|
|
215
|
+
# False for a job without categories. Overridden by batch, and by a job that holds the properties of another
|
|
216
|
+
# job, such as an UploadFileJob.
|
|
217
|
+
def proposes_schemas?
|
|
218
|
+
false
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options in the
|
|
222
|
+
# supplied properties of a job of this class, such as those that a Dirmon entry or an UploadFileJob holds,
|
|
223
|
+
# at the supplied place. None for a job without categories. Overridden by batch.
|
|
224
|
+
def property_schema_proposals(_properties, _where)
|
|
225
|
+
[]
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Returns [Hash] the supplied attributes or properties of this job class to show to people, with the value
|
|
229
|
+
# of each field that holds a path or url shown by its display name, without any credentials,
|
|
230
|
+
# see `path_fields`. Overridden by batch to also show the paths in its categories.
|
|
231
|
+
#
|
|
232
|
+
# For example, the properties that a Dirmon entry or an UploadFileJob holds for this job class.
|
|
233
|
+
def display_properties(properties)
|
|
234
|
+
properties.to_h do |key, value|
|
|
235
|
+
[key, path_fields.include?(key.to_sym) ? RocketJob.path_display_name(value) : value]
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options
|
|
241
|
+
# that this saved job holds, see RocketJob::SchemaProposal. None for a job without categories. Overridden by
|
|
242
|
+
# batch, and by a job that holds the properties of another job, such as an UploadFileJob.
|
|
243
|
+
def schema_proposals
|
|
244
|
+
[]
|
|
196
245
|
end
|
|
197
246
|
|
|
198
247
|
# Returns [Float] the number of seconds the job has taken
|
|
@@ -254,6 +303,12 @@ module RocketJob
|
|
|
254
303
|
run_at || created_at
|
|
255
304
|
end
|
|
256
305
|
|
|
306
|
+
# Returns [Hash] the attributes of this job to show to people, for example in Rocket Job Mission Control,
|
|
307
|
+
# with each path or url shown by its display name, without any credentials, see `.display_properties`.
|
|
308
|
+
def display_attributes
|
|
309
|
+
self.class.display_properties(attributes)
|
|
310
|
+
end
|
|
311
|
+
|
|
257
312
|
# Returns [Hash] status of this job
|
|
258
313
|
def as_json
|
|
259
314
|
attrs = serializable_hash(methods: %i[seconds duration])
|
|
@@ -23,6 +23,39 @@ module RocketJob
|
|
|
23
23
|
!@rails
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# Shown in place of a path that is not valid, since its credentials cannot be found to leave them out.
|
|
27
|
+
INVALID_PATH_DISPLAY_NAME = "(not a valid path)".freeze
|
|
28
|
+
|
|
29
|
+
# Returns [String] the supplied path or url to show, for example in a web interface, without any credentials,
|
|
30
|
+
# such as the user name and password of `sftp://user:password@host/file.csv`, see IOStreams::Path#display_name.
|
|
31
|
+
#
|
|
32
|
+
# A value that IOStreams cannot build a path from, such as an SFTP url with a non-ASCII letter or a `[` in its file
|
|
33
|
+
# name, or one that needs a gem that is not installed in this process, such as an S3 url without the aws-sdk-s3 gem,
|
|
34
|
+
# is shown as stored, see .display_text, since a url only holds credentials in its user name, password or query.
|
|
35
|
+
# Unless it holds an `@`, which ends a user name and password, or a `?`, which starts a query, when it is shown as
|
|
36
|
+
# INVALID_PATH_DISPLAY_NAME. An `@` anywhere in it counts, since a url that cannot be built can hold a password with
|
|
37
|
+
# a `/` or a `#`, such as `sftp://user:pa/ss@host/file.csv`.
|
|
38
|
+
def self.path_display_name(path)
|
|
39
|
+
return path if path.blank?
|
|
40
|
+
|
|
41
|
+
begin
|
|
42
|
+
IOStreams.path(path).display_name
|
|
43
|
+
rescue StandardError, LoadError
|
|
44
|
+
text = path.to_s
|
|
45
|
+
text.include?("@") || text.include?("?") ? INVALID_PATH_DISPLAY_NAME : display_text(text)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Returns [String] the supplied text as valid UTF-8, to show or save, with each byte that is not valid UTF-8, such
|
|
50
|
+
# as in a Latin-1 file name, shown as `\xHH`, the way that IOStreams::Path#display_name shows the name of a local
|
|
51
|
+
# file.
|
|
52
|
+
def self.display_text(text)
|
|
53
|
+
text = text.to_s.dup.force_encoding(Encoding::UTF_8)
|
|
54
|
+
return text if text.valid_encoding?
|
|
55
|
+
|
|
56
|
+
text.scrub { |bytes| bytes.unpack("C*").map { |byte| format("\\x%02X", byte) }.join }
|
|
57
|
+
end
|
|
58
|
+
|
|
26
59
|
# Returns a human readable duration from the supplied [Float] number of seconds
|
|
27
60
|
def self.seconds_as_duration(seconds)
|
|
28
61
|
return nil unless seconds
|