rocketjob 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/rocket_job/batch/categories.rb +161 -40
- data/lib/rocket_job/batch/io.rb +239 -133
- data/lib/rocket_job/category/base.rb +176 -1
- data/lib/rocket_job/category/input.rb +357 -41
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +39 -0
- data/lib/rocket_job/deprecation.rb +61 -0
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +114 -20
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/plugins/retry.rb +35 -34
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +13 -6
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +5 -0
- metadata +9 -4
|
@@ -23,6 +23,39 @@ module RocketJob
|
|
|
23
23
|
!@rails
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# Shown in place of a path that is not valid, since its credentials cannot be found to leave them out.
|
|
27
|
+
INVALID_PATH_DISPLAY_NAME = "(not a valid path)".freeze
|
|
28
|
+
|
|
29
|
+
# Returns [String] the supplied path or url to show, for example in a web interface, without any credentials,
|
|
30
|
+
# such as the user name and password of `sftp://user:password@host/file.csv`, see IOStreams::Path#display_name.
|
|
31
|
+
#
|
|
32
|
+
# A value that IOStreams cannot build a path from, such as an SFTP url with a non-ASCII letter or a `[` in its file
|
|
33
|
+
# name, or one that needs a gem that is not installed in this process, such as an S3 url without the aws-sdk-s3 gem,
|
|
34
|
+
# is shown as stored, see .display_text, since a url only holds credentials in its user name, password or query.
|
|
35
|
+
# Unless it holds an `@`, which ends a user name and password, or a `?`, which starts a query, when it is shown as
|
|
36
|
+
# INVALID_PATH_DISPLAY_NAME. An `@` anywhere in it counts, since a url that cannot be built can hold a password with
|
|
37
|
+
# a `/` or a `#`, such as `sftp://user:pa/ss@host/file.csv`.
|
|
38
|
+
def self.path_display_name(path)
|
|
39
|
+
return path if path.blank?
|
|
40
|
+
|
|
41
|
+
begin
|
|
42
|
+
IOStreams.path(path).display_name
|
|
43
|
+
rescue StandardError, LoadError
|
|
44
|
+
text = path.to_s
|
|
45
|
+
text.include?("@") || text.include?("?") ? INVALID_PATH_DISPLAY_NAME : display_text(text)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Returns [String] the supplied text as valid UTF-8, to show or save, with each byte that is not valid UTF-8, such
|
|
50
|
+
# as in a Latin-1 file name, shown as `\xHH`, the way that IOStreams::Path#display_name shows the name of a local
|
|
51
|
+
# file.
|
|
52
|
+
def self.display_text(text)
|
|
53
|
+
text = text.to_s.dup.force_encoding(Encoding::UTF_8)
|
|
54
|
+
return text if text.valid_encoding?
|
|
55
|
+
|
|
56
|
+
text.scrub { |bytes| bytes.unpack("C*").map { |byte| format("\\x%02X", byte) }.join }
|
|
57
|
+
end
|
|
58
|
+
|
|
26
59
|
# Returns a human readable duration from the supplied [Float] number of seconds
|
|
27
60
|
def self.seconds_as_duration(seconds)
|
|
28
61
|
return nil unless seconds
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
module RocketJob
|
|
2
|
+
# The schema proposed to replace the deprecated options of one category, see RocketJob::Category::Base#schema, of a
|
|
3
|
+
# job class, a saved job, a Dirmon entry or an UploadFileJob, so that each can be moved to a schema before Rocket Job
|
|
4
|
+
# 8 removes those options. A category proposes its own, see RocketJob::Category::Input#schema_proposal, from what
|
|
5
|
+
# its options did in Rocket Job 7.0, which is not always what they say.
|
|
6
|
+
#
|
|
7
|
+
# A proposal changes nothing: review it, read a recent file with it, see #dry_run, and apply it by hand, since a
|
|
8
|
+
# schema reads some files differently from the options that it replaces.
|
|
9
|
+
#
|
|
10
|
+
# Example, from a console of the application, once its job classes are loaded:
|
|
11
|
+
# puts RocketJob::SchemaProposal.report
|
|
12
|
+
class SchemaProposal
|
|
13
|
+
# What to check in the files of any category moved to a schema, since a schema reads them differently.
|
|
14
|
+
FILE_CHECKS = [
|
|
15
|
+
"A row with more values than the columns of its file raises IOStreams::Errors::InvalidHeader. Before, the " \
|
|
16
|
+
"values after the last column were ignored.",
|
|
17
|
+
"Two columns that cleanse to one name raise IOStreams::Errors::InvalidHeader. Before, the last one won.",
|
|
18
|
+
"Text that is not valid UTF-8 raises IOStreams::Errors::InvalidEncoding, unless an encoding is set on the " \
|
|
19
|
+
"path that is uploaded. Before, every character that was not ASCII was removed.",
|
|
20
|
+
"A column whose name only loosely matches a field, such as firstname for first_name, is read. Before, it was " \
|
|
21
|
+
"dropped.",
|
|
22
|
+
"The keys of each record are in the order of the fields. Before, they were in the order of the file."
|
|
23
|
+
].freeze
|
|
24
|
+
|
|
25
|
+
# The states of a saved job that can still run, and so read its categories.
|
|
26
|
+
RUNNABLE_JOB_STATES = %w[queued running paused failed].freeze
|
|
27
|
+
|
|
28
|
+
# Returns [String] where the category is, such as "ImportJob, input category main".
|
|
29
|
+
attr_reader :where
|
|
30
|
+
|
|
31
|
+
# Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category that the proposal is for, as it is.
|
|
32
|
+
attr_reader :category
|
|
33
|
+
|
|
34
|
+
# Returns [Hash] the settings that replace the deprecated options, by name, such as
|
|
35
|
+
# `{"allowed_columns" => nil, "schema" => {...}}`: a deprecated option is set to its default, and the schema to the
|
|
36
|
+
# one proposed. Empty when the category needs no change of its own, such as a job that is already running.
|
|
37
|
+
attr_reader :changes
|
|
38
|
+
|
|
39
|
+
# Returns [Array<String>] what to check before applying the proposal, or why there is none.
|
|
40
|
+
attr_reader :notes
|
|
41
|
+
|
|
42
|
+
# Returns [String|nil] why the deprecated options could not be translated into a schema, which then needs to be
|
|
43
|
+
# written by hand, or nil when they could.
|
|
44
|
+
attr_reader :error
|
|
45
|
+
|
|
46
|
+
def initialize(where:, category: nil, changes: {}, notes: [], error: nil)
|
|
47
|
+
@where = where
|
|
48
|
+
@category = category
|
|
49
|
+
@changes = changes
|
|
50
|
+
@notes = notes
|
|
51
|
+
@error = error
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Returns [Array<RocketJob::SchemaProposal>] the proposals for every batch job class that is loaded, see
|
|
55
|
+
# RocketJob::Batch::Categories::ClassMethods#schema_proposals, every saved job that can still run, of a class that
|
|
56
|
+
# proposes schemas or that is not loaded, see RocketJob::Plugins::Job::Model#schema_proposals and
|
|
57
|
+
# RocketJob::Plugins::Job::Model::ClassMethods#proposes_schemas?, and every Dirmon entry, see
|
|
58
|
+
# RocketJob::DirmonEntry#schema_proposals. In an application that loads its classes lazily, such as Rails in
|
|
59
|
+
# development, load them first, for example with `Rails.application.eager_load!`.
|
|
60
|
+
def self.all
|
|
61
|
+
proposals = []
|
|
62
|
+
RocketJob::Job.descendants.sort_by(&:to_s).each do |klass|
|
|
63
|
+
next unless klass.name && klass.respond_to?(:schema_proposals)
|
|
64
|
+
|
|
65
|
+
proposals.concat(owned_by(klass.name) { klass.schema_proposals })
|
|
66
|
+
end
|
|
67
|
+
# A saved job whose class is not loaded is read, to say that its class cannot be loaded.
|
|
68
|
+
without_proposals = [RocketJob::Job, *RocketJob::Job.descendants].reject(&:proposes_schemas?)
|
|
69
|
+
RocketJob::Job.collection.find(
|
|
70
|
+
"state" => {"$in" => RUNNABLE_JOB_STATES},
|
|
71
|
+
"_type" => {"$nin" => without_proposals.map(&:discriminator_value)}
|
|
72
|
+
).each do |document|
|
|
73
|
+
where = "Job #{document['_id']}"
|
|
74
|
+
proposals.concat(owned_by(where) { saved_job_proposals(document, where) })
|
|
75
|
+
end
|
|
76
|
+
RocketJob::DirmonEntry.each do |entry|
|
|
77
|
+
proposals.concat(owned_by("Dirmon entry #{entry.name.inspect} #{entry.id}") { entry.schema_proposals })
|
|
78
|
+
end
|
|
79
|
+
proposals
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Returns [Array<RocketJob::SchemaProposal>] the proposals of one owner, which the block returns, or one that says
|
|
83
|
+
# why its categories could not be read, such as a category property that its job class does not have, so that one
|
|
84
|
+
# of them does not stop the others from being listed.
|
|
85
|
+
def self.owned_by(where)
|
|
86
|
+
yield
|
|
87
|
+
rescue StandardError => e
|
|
88
|
+
[unreadable(where, e)]
|
|
89
|
+
end
|
|
90
|
+
private_class_method :owned_by
|
|
91
|
+
|
|
92
|
+
# Returns [RocketJob::SchemaProposal] a proposal at the supplied place that says why its categories could not be
|
|
93
|
+
# read, from the supplied exception, such as for a category property that its job class does not have. For an owner
|
|
94
|
+
# of proposals, such as a saved job or a Dirmon entry, to answer what its #schema_proposals raised.
|
|
95
|
+
def self.unreadable(where, exception)
|
|
96
|
+
new(where: where, error: "Its categories could not be read: #{exception.class}: #{exception.message}")
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Returns [String] the proposals, see .all, and what to check in the files of every category moved to a schema, to
|
|
100
|
+
# print, or "No deprecated category options were found." when there are none.
|
|
101
|
+
def self.report(proposals = all)
|
|
102
|
+
return "No deprecated category options were found." if proposals.empty?
|
|
103
|
+
|
|
104
|
+
checks = FILE_CHECKS.map { |check| " - #{check}" }
|
|
105
|
+
[*proposals.map(&:to_s), "For every category moved to a schema, check its files, see #dry_run:", *checks].join("\n\n")
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Returns [Array<RocketJob::SchemaProposal>] the proposals of the saved job that the supplied document holds, or one
|
|
109
|
+
# at the supplied place that says that its class cannot be loaded in this process. Each document is read on its own,
|
|
110
|
+
# so that one whose class cannot be loaded does not stop the others from being read. What else reading the job, or
|
|
111
|
+
# its proposals, raises is not rescued here, so that .all lists it as an error of the job.
|
|
112
|
+
def self.saved_job_proposals(document, where)
|
|
113
|
+
job = Mongoid::Factory.from_db(RocketJob::Job, document)
|
|
114
|
+
rescue Mongoid::Errors::UnknownModel
|
|
115
|
+
[new(where: where, notes: ["Its class cannot be loaded, so its categories cannot be read."])]
|
|
116
|
+
else
|
|
117
|
+
job.schema_proposals
|
|
118
|
+
end
|
|
119
|
+
private_class_method :saved_job_proposals
|
|
120
|
+
|
|
121
|
+
# Returns [Hash] the schema proposed, or nil when the deprecated options are removed without one, such as column
|
|
122
|
+
# options that never applied to JSON records.
|
|
123
|
+
def schema
|
|
124
|
+
changes["schema"]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Returns [String] the proposal to show to people: where it is, the settings that replace the deprecated options, and
|
|
128
|
+
# what to check before applying them.
|
|
129
|
+
def to_s
|
|
130
|
+
lines = [where]
|
|
131
|
+
lines << " Cannot translate: #{error}" if error
|
|
132
|
+
unless changes.empty?
|
|
133
|
+
removed = changes.except("schema").map { |name, value| value.nil? ? "#{name}: remove" : "#{name}: #{value.inspect}" }
|
|
134
|
+
lines << " Set: #{removed.join(', ')}" unless removed.empty?
|
|
135
|
+
lines << " Set schema: #{schema.inspect}" if schema
|
|
136
|
+
end
|
|
137
|
+
notes.each { |note| lines << " Check: #{note}" }
|
|
138
|
+
lines.join("\n")
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Returns [Array<String>] how the first records of the supplied file differ when they are read with this
|
|
142
|
+
# proposal, from how its category reads them now, see RocketJob::Category::Input#preview, such as a key that is
|
|
143
|
+
# missing, or a value or error that changed. Empty when they are the same. Changes nothing.
|
|
144
|
+
#
|
|
145
|
+
# The file is read twice, so supply its file name or IOStreams path, or a stream that responds to #rewind.
|
|
146
|
+
#
|
|
147
|
+
# Raises ArgumentError for a proposal that is not for an input category, or that has no changes.
|
|
148
|
+
def dry_run(stream, limit: 100)
|
|
149
|
+
unless category.is_a?(RocketJob::Category::Input) && changes.any?
|
|
150
|
+
raise(ArgumentError, "Only a proposal that changes an input category can read a file")
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
before = read(category, stream, limit)
|
|
154
|
+
after = read(proposed_category, stream, limit)
|
|
155
|
+
compare(before, after)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Returns [RocketJob::Category::Input|RocketJob::Category::Output] a copy of the category with the proposed
|
|
159
|
+
# changes, see #changes.
|
|
160
|
+
def proposed_category
|
|
161
|
+
copy = category.deep_dup
|
|
162
|
+
changes.except("schema").each_pair { |name, value| copy.public_send(:"#{name}=", value) }
|
|
163
|
+
copy.schema = schema if schema
|
|
164
|
+
copy
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
private
|
|
168
|
+
|
|
169
|
+
# Returns [Array|Exception] the records of the file read by the category, or what reading it raised.
|
|
170
|
+
def read(category, stream, limit)
|
|
171
|
+
# A stream, such as a StringIO, is read from its start each time.
|
|
172
|
+
stream.rewind if stream.respond_to?(:rewind)
|
|
173
|
+
category.preview(stream, limit: limit)
|
|
174
|
+
rescue StandardError => e
|
|
175
|
+
e
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def compare(before, after)
|
|
179
|
+
return [] if in_order(before) == in_order(after)
|
|
180
|
+
return [compare_errors(before, after)] if before.is_a?(Exception) || after.is_a?(Exception)
|
|
181
|
+
|
|
182
|
+
differences = []
|
|
183
|
+
differences << "Reads #{after.size} records rather than #{before.size}" if before.size != after.size
|
|
184
|
+
before.zip(after).each_with_index do |(old, new), index|
|
|
185
|
+
difference = compare_record(old, new)
|
|
186
|
+
differences << "Record #{index + 1}: #{difference}" if difference
|
|
187
|
+
end
|
|
188
|
+
differences
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Returns [String] how what the proposal raises differs from what the category raises, or reads, now.
|
|
192
|
+
def compare_errors(before, after)
|
|
193
|
+
return "Now raises #{after.class}: #{after.message}" unless before.is_a?(Exception)
|
|
194
|
+
return "No longer raises #{before.class}: #{before.message}" unless after.is_a?(Exception)
|
|
195
|
+
return "Raises #{after.class} rather than #{before.class}: #{after.message}" unless after.instance_of?(before.class)
|
|
196
|
+
return "Raises #{after.class}, as it does now: #{after.message}" if after.message == before.message
|
|
197
|
+
|
|
198
|
+
"Raises #{after.class} with another message: #{after.message}, rather than: #{before.message}"
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def in_order(records)
|
|
202
|
+
return records unless records.is_a?(Array)
|
|
203
|
+
|
|
204
|
+
records.map { |record| in_order_record(record) }
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Hashes are equal whatever the order of their keys, which a record's keys keep.
|
|
208
|
+
def in_order_record(record)
|
|
209
|
+
record.is_a?(Hash) ? record.to_a : record
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def compare_record(old, new)
|
|
213
|
+
return if in_order_record(old) == in_order_record(new)
|
|
214
|
+
return "#{new.inspect} rather than #{old.inspect}" unless old.is_a?(Hash) && new.is_a?(Hash)
|
|
215
|
+
return "keys in another order: #{new.keys.inspect} rather than #{old.keys.inspect}" if old == new
|
|
216
|
+
|
|
217
|
+
compare_keys(old, new).join("; ")
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def compare_keys(old, new)
|
|
221
|
+
parts = []
|
|
222
|
+
missing = old.keys - new.keys
|
|
223
|
+
added = new.keys - old.keys
|
|
224
|
+
parts << "keys no longer read: #{missing.inspect}" unless missing.empty?
|
|
225
|
+
parts << "new keys: #{added.inspect}" unless added.empty?
|
|
226
|
+
(old.keys & new.keys).each do |key|
|
|
227
|
+
parts << "#{key.inspect} is #{new[key].inspect} rather than #{old[key].inspect}" if old[key] != new[key]
|
|
228
|
+
end
|
|
229
|
+
parts
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
@@ -1,16 +1,35 @@
|
|
|
1
1
|
module RocketJob
|
|
2
2
|
module Sliced
|
|
3
3
|
class Input < Slices
|
|
4
|
+
# Uploads the records that the block writes, see Writer::Input.collect, and returns how many it wrote.
|
|
5
|
+
#
|
|
6
|
+
# When the upload raises, its slices are removed, and the slices of earlier uploads are kept: when the collection
|
|
7
|
+
# already holds slices, this upload gives its own an id, see Slice#upload_id, otherwise the whole collection is
|
|
8
|
+
# dropped, so that the slices of the first upload do not each hold an id.
|
|
4
9
|
def upload(**args, &)
|
|
5
|
-
# Create indexes before uploading
|
|
10
|
+
# Create indexes before uploading, outside the rescue below, so that an error here does not drop the slices of
|
|
11
|
+
# earlier uploads.
|
|
6
12
|
create_indexes
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
13
|
+
upload_id = BSON::ObjectId.new.to_s if exists?
|
|
14
|
+
begin
|
|
15
|
+
Writer::Input.collect(self, upload_id: upload_id, **args, &)
|
|
16
|
+
rescue Exception => e
|
|
17
|
+
upload_id ? delete_upload(upload_id) : drop
|
|
18
|
+
raise(e)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Removes the slices of the upload with the supplied id, see Slice#upload_id.
|
|
23
|
+
def delete_upload(upload_id)
|
|
24
|
+
where(upload_id: upload_id).delete_all
|
|
11
25
|
end
|
|
12
26
|
|
|
27
|
+
# Uploads the documents of the supplied Mongoid criteria, see Batch::IO#upload, and returns how many it uploaded.
|
|
28
|
+
#
|
|
29
|
+
# Reports a deprecation warning, see QueryDeprecation, for what Rocket Job 8 uploads differently, without
|
|
30
|
+
# changing what this upload stores.
|
|
13
31
|
def upload_mongo_query(criteria, columns: [], slice_batch_size: nil, &block)
|
|
32
|
+
QueryDeprecation.warn_criteria(block)
|
|
14
33
|
options = criteria.options
|
|
15
34
|
|
|
16
35
|
# Without a block extract the fields from the supplied criteria
|
|
@@ -40,7 +59,12 @@ module RocketJob
|
|
|
40
59
|
end
|
|
41
60
|
end
|
|
42
61
|
|
|
62
|
+
# Uploads the rows of the supplied Active Record relation, see Batch::IO#upload, and returns how many it uploaded.
|
|
63
|
+
#
|
|
64
|
+
# Reports a deprecation warning, see QueryDeprecation, for what Rocket Job 8 uploads differently, without
|
|
65
|
+
# changing what this upload stores.
|
|
43
66
|
def upload_arel(arel, columns: nil, slice_batch_size: nil, &block)
|
|
67
|
+
QueryDeprecation.warn_relation(arel, block)
|
|
44
68
|
unless block
|
|
45
69
|
columns = columns.blank? ? [:id] : columns.collect(&:to_sym)
|
|
46
70
|
|
|
@@ -51,6 +75,7 @@ module RocketJob
|
|
|
51
75
|
else
|
|
52
76
|
->(model) { columns.collect { |c| model.public_send(c) } }
|
|
53
77
|
end
|
|
78
|
+
block = QueryDeprecation.check_readers(columns, block)
|
|
54
79
|
# find_each requires the :id column in the query
|
|
55
80
|
selection = columns.include?(:id) ? columns : columns + [:id]
|
|
56
81
|
arel = arel.select(selection)
|
|
@@ -138,6 +163,125 @@ module RocketJob
|
|
|
138
163
|
document.collection_name = collection_name if document
|
|
139
164
|
document
|
|
140
165
|
end
|
|
166
|
+
|
|
167
|
+
# Knows what Rocket Job 8 changes in uploading a query, and reports a deprecation warning, see
|
|
168
|
+
# RocketJob::Deprecation, for each change that an upload depends on, so that the upload methods above only upload
|
|
169
|
+
# as 7.x does, and store exactly what they stored before.
|
|
170
|
+
#
|
|
171
|
+
# Rocket Job 8 reads an Active Record relation in one streamed query, with each value cast as `pluck` casts it,
|
|
172
|
+
# and builds no models, so:
|
|
173
|
+
# - A block passed with a query, a relation or a Mongoid criteria, raises ArgumentError: the block of `upload`
|
|
174
|
+
# writes the records instead, such as those of a query that it reads itself.
|
|
175
|
+
# - A relation that eager loads its associations, with `includes`, `preload` or `eager_load`, raises
|
|
176
|
+
# ArgumentError.
|
|
177
|
+
# - Each column uploads the value that it holds, as `pluck` returns it, rather than calling the reader of the
|
|
178
|
+
# model, which differs only when the model overrides the reader.
|
|
179
|
+
#
|
|
180
|
+
# The records of one column, the value of each row, and of several, an Array of their values, are unchanged.
|
|
181
|
+
#
|
|
182
|
+
# An upload reports each change that it depends on once, and the reader of each column at most once.
|
|
183
|
+
class QueryDeprecation
|
|
184
|
+
# The methods of a relation that eager load its associations, which Rocket Job 8 cannot upload.
|
|
185
|
+
EAGER_LOADING = %i[includes preload eager_load].freeze
|
|
186
|
+
|
|
187
|
+
# Reports a deprecation warning for each change that the upload of the supplied Active Record relation, or
|
|
188
|
+
# model class, depends on: that it eager loads associations, and that the supplied block is passed with it.
|
|
189
|
+
#
|
|
190
|
+
# Returns nil.
|
|
191
|
+
def self.warn_relation(relation, block)
|
|
192
|
+
warn_eager_loading(relation)
|
|
193
|
+
warn_block("an Active Record relation", "User.where(country_code: \"US\").find_each") if block
|
|
194
|
+
nil
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Reports a deprecation warning when the supplied block is passed with the upload of a Mongoid criteria.
|
|
198
|
+
#
|
|
199
|
+
# Returns nil.
|
|
200
|
+
def self.warn_criteria(block)
|
|
201
|
+
warn_block("a Mongoid criteria", "User.where(state: \"FL\").each") if block
|
|
202
|
+
nil
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Reports a deprecation warning that a block passed with the supplied kind of query raises in Rocket Job 8, with
|
|
206
|
+
# an example that reads the query in the block of `upload` instead.
|
|
207
|
+
def self.warn_block(kind, query)
|
|
208
|
+
RocketJob::Deprecation.warn(
|
|
209
|
+
"a block passed with #{kind} raises ArgumentError in Rocket Job 8, which uploads the values of its " \
|
|
210
|
+
"columns: write the records in the block of upload instead, such as " \
|
|
211
|
+
"job.upload { |records| #{query} { |user| records << user.email.downcase } }, see " \
|
|
212
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
213
|
+
)
|
|
214
|
+
end
|
|
215
|
+
private_class_method :warn_block
|
|
216
|
+
|
|
217
|
+
# Returns [Proc] the supplied block, which returns the record that 7.x uploads for a model, while deprecation
|
|
218
|
+
# warnings are off, so that production does not pay for checking every model. Otherwise returns a Proc that
|
|
219
|
+
# also checks each model, after the block has read it, for a reader of one of the supplied columns that
|
|
220
|
+
# returns a value other than the one that the column holds, see #check.
|
|
221
|
+
def self.check_readers(columns, block)
|
|
222
|
+
return block unless RocketJob::Deprecation.enabled?
|
|
223
|
+
|
|
224
|
+
readers = new(columns)
|
|
225
|
+
lambda do |model|
|
|
226
|
+
record = block.call(model)
|
|
227
|
+
readers.check(model)
|
|
228
|
+
record
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Reports a deprecation warning, naming the methods, when the supplied relation, or model class, such as one
|
|
233
|
+
# with a default scope, eager loads its associations.
|
|
234
|
+
def self.warn_eager_loading(relation)
|
|
235
|
+
return unless relation.respond_to?(:all)
|
|
236
|
+
|
|
237
|
+
scope = relation.all
|
|
238
|
+
methods = EAGER_LOADING.select { |name| scope.public_send(:"#{name}_values").present? }
|
|
239
|
+
return if methods.empty?
|
|
240
|
+
|
|
241
|
+
# Without the connectors of the application's locale, so that the message is the same in every application.
|
|
242
|
+
names = methods.to_sentence(last_word_connector: " and ", locale: false)
|
|
243
|
+
RocketJob::Deprecation.warn(
|
|
244
|
+
"uploading an Active Record relation that uses #{names} raises ArgumentError in Rocket Job 8, which reads " \
|
|
245
|
+
"the rows of one query and builds no models: join the tables with joins and select their columns instead, " \
|
|
246
|
+
"or to filter by an association, use joins with distinct, or a subquery, see " \
|
|
247
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
248
|
+
)
|
|
249
|
+
end
|
|
250
|
+
private_class_method :warn_eager_loading
|
|
251
|
+
|
|
252
|
+
# The supplied columns, whose readers are checked.
|
|
253
|
+
def initialize(columns)
|
|
254
|
+
@columns = columns.dup
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Reports a deprecation warning for each column whose reader on the supplied model returns a value other than
|
|
258
|
+
# the one that the column holds, see ActiveRecord::AttributeMethods::Read#read_attribute, which is what `pluck`
|
|
259
|
+
# returns, and what Rocket Job 8 uploads. Then stops checking that column, so that each column warns at most
|
|
260
|
+
# once. Names the model class and the column, but never the values, which could be personal data.
|
|
261
|
+
#
|
|
262
|
+
# The reader is called again to compare it, after the upload has read the model, so the record uploaded is the
|
|
263
|
+
# one that the upload would store without the check. A reader that returns the value itself is the same, such
|
|
264
|
+
# as a Float NaN, which is not equal to itself.
|
|
265
|
+
#
|
|
266
|
+
# Returns nil.
|
|
267
|
+
def check(model)
|
|
268
|
+
@columns.reject! do |column|
|
|
269
|
+
value = model.public_send(column)
|
|
270
|
+
stored = model.read_attribute(column)
|
|
271
|
+
next false if value.equal?(stored) || value == stored
|
|
272
|
+
|
|
273
|
+
RocketJob::Deprecation.warn(
|
|
274
|
+
"#{model.class}##{column} returns a value other than the one that its column holds, which Rocket Job 8 " \
|
|
275
|
+
"uploads instead, as pluck returns it, rather than calling the reader: to keep the reader's value, write " \
|
|
276
|
+
"the records in the block of upload, which can call it, or apply the reader's logic in perform, see " \
|
|
277
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
278
|
+
)
|
|
279
|
+
true
|
|
280
|
+
end
|
|
281
|
+
nil
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
private_constant :QueryDeprecation
|
|
141
285
|
end
|
|
142
286
|
end
|
|
143
287
|
end
|
|
@@ -47,6 +47,10 @@ module RocketJob
|
|
|
47
47
|
# This name of the worker that this job is being processed by, or was processed by
|
|
48
48
|
field :worker_name, type: String
|
|
49
49
|
|
|
50
|
+
# The id of the upload that inserted this slice, when the input already held the slices of an earlier upload, so
|
|
51
|
+
# that the slices of this upload can be removed again when it fails, see RocketJob::Sliced::Input#upload.
|
|
52
|
+
field :upload_id, type: String
|
|
53
|
+
|
|
50
54
|
# The last exception for this slice if any
|
|
51
55
|
embeds_one :exception, class_name: "RocketJob::JobException"
|
|
52
56
|
|
|
@@ -120,7 +120,7 @@ module RocketJob
|
|
|
120
120
|
end
|
|
121
121
|
|
|
122
122
|
# Forward additional methods.
|
|
123
|
-
def_instance_delegators :@all, :collection, :count, :delete_all, :find, :nor, :not, :or, :to_a, :where
|
|
123
|
+
def_instance_delegators :@all, :collection, :count, :delete_all, :exists?, :find, :nor, :not, :or, :to_a, :where
|
|
124
124
|
|
|
125
125
|
# Drop this collection when it is no longer needed
|
|
126
126
|
def drop
|
|
@@ -9,8 +9,9 @@ module RocketJob
|
|
|
9
9
|
#
|
|
10
10
|
# Parameters
|
|
11
11
|
# on_first: [Proc]
|
|
12
|
-
#
|
|
13
|
-
# Useful for extracting the header row
|
|
12
|
+
# Called with each line before the first that is stored, instead of storing it, until it returns true.
|
|
13
|
+
# Useful for extracting the header row, after any blank lines before it, see
|
|
14
|
+
# Category::Input#extract_header_callback.
|
|
14
15
|
# Default: nil
|
|
15
16
|
#
|
|
16
17
|
# slice_size: [Integer]
|
|
@@ -21,6 +22,10 @@ module RocketJob
|
|
|
21
22
|
# The number of slices to batch up and to bulk load.
|
|
22
23
|
# For smaller slices this significantly improves upload performance.
|
|
23
24
|
# Note: If `slice_batch_size` is too high, it can exceed the maximum BSON block size.
|
|
25
|
+
#
|
|
26
|
+
# upload_id: [String]
|
|
27
|
+
# The id of this upload, which each slice holds, see Slice#upload_id.
|
|
28
|
+
# Default: nil
|
|
24
29
|
def self.collect(data_store, **args)
|
|
25
30
|
writer = new(data_store, **args)
|
|
26
31
|
yield(writer)
|
|
@@ -29,8 +34,9 @@ module RocketJob
|
|
|
29
34
|
writer&.flush
|
|
30
35
|
end
|
|
31
36
|
|
|
32
|
-
def initialize(data_store, on_first: nil, slice_size: nil, slice_batch_size: nil)
|
|
37
|
+
def initialize(data_store, on_first: nil, slice_size: nil, slice_batch_size: nil, upload_id: nil)
|
|
33
38
|
@on_first = on_first
|
|
39
|
+
@upload_id = upload_id
|
|
34
40
|
@record_count = 0
|
|
35
41
|
@data_store = data_store
|
|
36
42
|
@slice_size = slice_size || @data_store.slice_size
|
|
@@ -42,8 +48,7 @@ module RocketJob
|
|
|
42
48
|
|
|
43
49
|
def <<(line)
|
|
44
50
|
if @on_first
|
|
45
|
-
@on_first.call(line)
|
|
46
|
-
@on_first = nil
|
|
51
|
+
@on_first = nil if @on_first.call(line)
|
|
47
52
|
return self
|
|
48
53
|
end
|
|
49
54
|
@slice << line
|
|
@@ -67,7 +72,9 @@ module RocketJob
|
|
|
67
72
|
end
|
|
68
73
|
|
|
69
74
|
def new_slice
|
|
70
|
-
|
|
75
|
+
attributes = {first_record_number: @record_count + 1}
|
|
76
|
+
attributes[:upload_id] = @upload_id if @upload_id
|
|
77
|
+
@slice = @data_store.new(attributes)
|
|
71
78
|
end
|
|
72
79
|
|
|
73
80
|
def save_slice
|
data/lib/rocket_job/version.rb
CHANGED
data/lib/rocketjob.rb
CHANGED
|
@@ -23,6 +23,7 @@ module RocketJob
|
|
|
23
23
|
autoload :ActiveWorker, "rocket_job/active_worker"
|
|
24
24
|
autoload :Batch, "rocket_job/batch"
|
|
25
25
|
autoload :CLI, "rocket_job/cli"
|
|
26
|
+
autoload :Deprecation, "rocket_job/deprecation"
|
|
26
27
|
autoload :DirmonEntry, "rocket_job/dirmon_entry"
|
|
27
28
|
autoload :Event, "rocket_job/event"
|
|
28
29
|
autoload :Heartbeat, "rocket_job/heartbeat"
|
|
@@ -31,6 +32,7 @@ module RocketJob
|
|
|
31
32
|
autoload :LookupCollection, "rocket_job/lookup_collection"
|
|
32
33
|
autoload :Worker, "rocket_job/worker"
|
|
33
34
|
autoload :Performance, "rocket_job/performance"
|
|
35
|
+
autoload :SchemaProposal, "rocket_job/schema_proposal"
|
|
34
36
|
autoload :Server, "rocket_job/server"
|
|
35
37
|
autoload :Sliced, "rocket_job/sliced"
|
|
36
38
|
autoload :Subscriber, "rocket_job/subscriber"
|
|
@@ -43,6 +45,9 @@ module RocketJob
|
|
|
43
45
|
module Category
|
|
44
46
|
autoload :Base, "rocket_job/category/base"
|
|
45
47
|
autoload :Input, "rocket_job/category/input"
|
|
48
|
+
autoload :InputSchemaProposal, "rocket_job/category/input_schema_proposal"
|
|
49
|
+
autoload :InputSchemaHeader, "rocket_job/category/input_schema_header"
|
|
50
|
+
autoload :InputTextLines, "rocket_job/category/input_text_lines"
|
|
46
51
|
autoload :Output, "rocket_job/category/output"
|
|
47
52
|
end
|
|
48
53
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rocketjob
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 7.
|
|
4
|
+
version: 7.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Reid Morrison
|
|
@@ -57,14 +57,14 @@ dependencies:
|
|
|
57
57
|
requirements:
|
|
58
58
|
- - "~>"
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '2.
|
|
60
|
+
version: '2.2'
|
|
61
61
|
type: :runtime
|
|
62
62
|
prerelease: false
|
|
63
63
|
version_requirements: !ruby/object:Gem::Requirement
|
|
64
64
|
requirements:
|
|
65
65
|
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '2.
|
|
67
|
+
version: '2.2'
|
|
68
68
|
- !ruby/object:Gem::Dependency
|
|
69
69
|
name: mongoid
|
|
70
70
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -142,9 +142,13 @@ files:
|
|
|
142
142
|
- lib/rocket_job/batch/worker.rb
|
|
143
143
|
- lib/rocket_job/category/base.rb
|
|
144
144
|
- lib/rocket_job/category/input.rb
|
|
145
|
+
- lib/rocket_job/category/input_schema_header.rb
|
|
146
|
+
- lib/rocket_job/category/input_schema_proposal.rb
|
|
147
|
+
- lib/rocket_job/category/input_text_lines.rb
|
|
145
148
|
- lib/rocket_job/category/output.rb
|
|
146
149
|
- lib/rocket_job/cli.rb
|
|
147
150
|
- lib/rocket_job/config.rb
|
|
151
|
+
- lib/rocket_job/deprecation.rb
|
|
148
152
|
- lib/rocket_job/dirmon_entry.rb
|
|
149
153
|
- lib/rocket_job/event.rb
|
|
150
154
|
- lib/rocket_job/extensions/iostreams/path.rb
|
|
@@ -190,6 +194,7 @@ files:
|
|
|
190
194
|
- lib/rocket_job/plugins/transaction.rb
|
|
191
195
|
- lib/rocket_job/railtie.rb
|
|
192
196
|
- lib/rocket_job/rocket_job.rb
|
|
197
|
+
- lib/rocket_job/schema_proposal.rb
|
|
193
198
|
- lib/rocket_job/server.rb
|
|
194
199
|
- lib/rocket_job/server/model.rb
|
|
195
200
|
- lib/rocket_job/server/state_machine.rb
|
|
@@ -226,7 +231,7 @@ metadata:
|
|
|
226
231
|
changelog_uri: https://github.com/reidmorrison/rocketjob/blob/main/CHANGELOG.md
|
|
227
232
|
documentation_uri: https://rocketjob.reidmorrison.com
|
|
228
233
|
homepage_uri: https://rocketjob.reidmorrison.com
|
|
229
|
-
source_code_uri: https://github.com/reidmorrison/rocketjob/tree/v7.
|
|
234
|
+
source_code_uri: https://github.com/reidmorrison/rocketjob/tree/v7.2.0
|
|
230
235
|
rubygems_mfa_required: 'true'
|
|
231
236
|
rdoc_options: []
|
|
232
237
|
require_paths:
|