rocketjob 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/rocket_job/batch/categories.rb +161 -40
- data/lib/rocket_job/batch/io.rb +239 -133
- data/lib/rocket_job/category/base.rb +176 -1
- data/lib/rocket_job/category/input.rb +357 -41
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +39 -0
- data/lib/rocket_job/deprecation.rb +61 -0
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +114 -20
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/plugins/retry.rb +35 -34
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +13 -6
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +5 -0
- metadata +9 -4
data/lib/rocket_job/batch/io.rb
CHANGED
|
@@ -80,6 +80,7 @@ module RocketJob
|
|
|
80
80
|
# Or, an IOStreams path such as IOStreams::Paths::File, or IOStreams::Paths::S3
|
|
81
81
|
#
|
|
82
82
|
# delimiter[String]
|
|
83
|
+
# Deprecated, and removed in Rocket Job 8, which detects the line endings of a file.
|
|
83
84
|
# Line / Record delimiter to use to break the stream up into records
|
|
84
85
|
# Any string to break the stream up by
|
|
85
86
|
# The records when saved will not include this delimiter
|
|
@@ -88,7 +89,20 @@ module RocketJob
|
|
|
88
89
|
# Searches for the first "\r\n" or "\n" and then uses that as the
|
|
89
90
|
# delimiter for all subsequent records
|
|
90
91
|
#
|
|
92
|
+
# validate_rows: [true | false]
|
|
93
|
+
# true to parse each line of the file in the input category's format while it is uploaded, so that a file
|
|
94
|
+
# with a line that cannot be read, or with a value that is not valid, uploads nothing. The lines are still
|
|
95
|
+
# uploaded as they are, and the workers parse them.
|
|
96
|
+
# Raises ArgumentError when true for an input category without a format, or for a query, a range or a block,
|
|
97
|
+
# which upload values rather than the lines of a file, for which false is accepted, since it validates
|
|
98
|
+
# nothing. Also raises when supplied, true or false, with a `stream_mode:` other than nil or false, which
|
|
99
|
+
# decides alone whether the lines are validated.
|
|
100
|
+
# Default: The input category's `validate_rows`, see RocketJob::Category::Input#upload_validates_rows?.
|
|
101
|
+
#
|
|
91
102
|
# stream_mode: [:line | :array | :hash]
|
|
103
|
+
# Deprecated, and removed in Rocket Job 8, which raises ArgumentError whenever it is supplied, also as nil, so
|
|
104
|
+
# it reports a deprecation warning then too: `validate_rows: true` replaces :hash, and for an input category
|
|
105
|
+
# without a format, its `format: :auto` reads a file into Hashes.
|
|
92
106
|
# :line
|
|
93
107
|
# Uploads the file a line (String) at a time for processing by workers.
|
|
94
108
|
# :array
|
|
@@ -109,7 +123,7 @@ module RocketJob
|
|
|
109
123
|
# Example:
|
|
110
124
|
# # Load plain text records from a file, stripping all non-printable characters,
|
|
111
125
|
# # as well as any characters that cannot be converted to UTF-8
|
|
112
|
-
# path = IOStreams.path('hello.csv').
|
|
126
|
+
# path = IOStreams.path('hello.csv').encoding(cleaner: :printable, replace: '')
|
|
113
127
|
# job.upload(path)
|
|
114
128
|
#
|
|
115
129
|
# Example: Zip
|
|
@@ -120,11 +134,11 @@ module RocketJob
|
|
|
120
134
|
# job.upload('myfile.csv.zip.enc')
|
|
121
135
|
#
|
|
122
136
|
# Example: Explicitly set the streams
|
|
123
|
-
# path = IOStreams.path('myfile.ze').stream(:
|
|
137
|
+
# path = IOStreams.path('myfile.ze').stream(:zip).stream(:enc).encoding('UTF-8')
|
|
124
138
|
# job.upload(path)
|
|
125
139
|
#
|
|
126
140
|
# Example: Supply custom options
|
|
127
|
-
# path = IOStreams.path('myfile.csv.enc').option(:enc, compress: false).
|
|
141
|
+
# path = IOStreams.path('myfile.csv.enc').option(:enc, compress: false).encoding('UTF-8')
|
|
128
142
|
# job.upload(path)
|
|
129
143
|
#
|
|
130
144
|
# Example: Read from a tempfile and use the original file name to determine which streams to apply
|
|
@@ -140,141 +154,93 @@ module RocketJob
|
|
|
140
154
|
#
|
|
141
155
|
# Notes:
|
|
142
156
|
# * Only call from one thread at a time against a single instance of this job.
|
|
143
|
-
# * The
|
|
144
|
-
# * If an exception is raised while uploading data, the
|
|
145
|
-
#
|
|
146
|
-
# *
|
|
147
|
-
#
|
|
157
|
+
# * The number of records uploaded is added to the record_count of the job.
|
|
158
|
+
# * If an exception is raised while uploading data, the records that this upload stored are removed, so that
|
|
159
|
+
# if the job is retried, its data is not duplicated. The records of earlier uploads are kept.
|
|
160
|
+
# * The text of the file/stream is read as UTF-8 before being persisted, since MongoDB only stores UTF-8 strings.
|
|
161
|
+
# Text that is not valid UTF-8 raises IOStreams::Errors::InvalidEncoding, naming its line, unless an encoding
|
|
162
|
+
# is set on the path, such as `IOStreams.path("names.txt").encoding("ISO-8859-1")`, which reads it instead.
|
|
163
|
+
# An input category with a format but no schema ignores an encoding set on the path, and removes
|
|
164
|
+
# non-printable characters, and every character that is not ASCII, instead.
|
|
148
165
|
# * When zip format, the Zip file/stream must contain only one file, the first file found will be
|
|
149
166
|
# loaded into the job
|
|
150
167
|
# * If an io stream is supplied, it is read until it returns nil.
|
|
151
168
|
# * Only use this method for UTF-8 data, for binary data use #input_slice or #input_records.
|
|
152
169
|
# * CSV parsing is slow, so it is usually left for the workers to do.
|
|
153
170
|
#
|
|
154
|
-
# Upload
|
|
155
|
-
#
|
|
156
|
-
# Params
|
|
157
|
-
# column_names
|
|
158
|
-
# When a block is not supplied, supply the names of the columns to be returned
|
|
159
|
-
# and uploaded into the job
|
|
160
|
-
# These columns are automatically added to the select list to reduce overhead
|
|
161
|
-
#
|
|
162
|
-
# category [Symbol|RocketJob::Category::Input]
|
|
163
|
-
# The category or the name of the category to upload to.
|
|
164
|
-
#
|
|
165
|
-
# If a Block is supplied it is passed the model returned from the database and should
|
|
166
|
-
# return the work item to be uploaded into the job.
|
|
167
|
-
#
|
|
168
|
-
# Returns [Integer] the number of records uploaded
|
|
169
|
-
#
|
|
170
|
-
# Example: Upload id's for all users
|
|
171
|
-
# arel = User.all
|
|
172
|
-
# job.upload_arel(arel)
|
|
173
|
-
#
|
|
174
|
-
# Example: Upload selected user id's
|
|
175
|
-
# arel = User.where(country_code: 'US')
|
|
176
|
-
# job.upload_arel(arel)
|
|
177
|
-
#
|
|
178
|
-
# Example: Upload user_name and zip_code
|
|
179
|
-
# arel = User.where(country_code: 'US')
|
|
180
|
-
# job.upload_arel(arel, columns: [:user_name, :zip_code])
|
|
181
|
-
#
|
|
182
|
-
# Notes:
|
|
183
|
-
# * Only call from one thread at a time against a single instance of this job.
|
|
184
|
-
# * The record_count for the job is set to the number of records returned by the arel.
|
|
185
|
-
# * If an exception is raised while uploading data, the input collection is cleared out
|
|
186
|
-
# so that if a job is retried during an upload failure, data is not duplicated.
|
|
187
|
-
|
|
188
|
-
# Upload the result of a MongoDB query to the input collection for processing
|
|
189
|
-
# Useful when an entire MongoDB collection, or part thereof needs to be
|
|
190
|
-
# processed by a job.
|
|
191
|
-
#
|
|
192
|
-
# Returns [Integer] the number of records uploaded
|
|
193
|
-
#
|
|
194
|
-
# If a Block is supplied it is passed the document returned from the
|
|
195
|
-
# database and should return a record for processing
|
|
196
|
-
#
|
|
197
|
-
# If no Block is supplied then the record will be the :fields returned
|
|
198
|
-
# from MongoDB
|
|
171
|
+
# Upload the records of a query or a range of integers, which replaces upload_arel, upload_mongo_query,
|
|
172
|
+
# upload_integer_range and upload_integer_range_in_reverse_order, that Rocket Job 8 removes:
|
|
199
173
|
#
|
|
200
|
-
#
|
|
201
|
-
#
|
|
202
|
-
# avoid the overhead of constructing a Model with every document returned
|
|
203
|
-
# by the query
|
|
204
|
-
#
|
|
205
|
-
# Note:
|
|
206
|
-
# The Block must return types that can be serialized to BSON.
|
|
207
|
-
# Valid Types: Hash | Array | String | Integer | Float | Symbol | Regexp | Time
|
|
208
|
-
# Invalid: Date, etc.
|
|
174
|
+
# Example: Upload the id of each user that an Active Record relation returns
|
|
175
|
+
# job.upload(User.where(country_code: "US"))
|
|
209
176
|
#
|
|
210
|
-
# Example: Upload
|
|
211
|
-
#
|
|
212
|
-
# job.record_count = job.upload_mongo_query(criteria)
|
|
177
|
+
# Example: Upload the user_name and zip_code of each user, as an Array, see `columns:`
|
|
178
|
+
# job.upload(User.where(country_code: "US"), columns: %i[user_name zip_code])
|
|
213
179
|
#
|
|
214
|
-
# Example: Upload
|
|
215
|
-
#
|
|
216
|
-
#
|
|
217
|
-
#
|
|
218
|
-
#
|
|
219
|
-
#
|
|
220
|
-
# * The record_count for the job is set to the number of records returned by the monqo query.
|
|
221
|
-
# * If an exception is raised while uploading data, the input collection is cleared out
|
|
222
|
-
# so that if a job is retried during an upload failure, data is not duplicated.
|
|
223
|
-
|
|
224
|
-
# Upload sliced range of integer requests as arrays of start and end ids.
|
|
225
|
-
#
|
|
226
|
-
# Returns [Integer] the number of slices uploaded.
|
|
180
|
+
# Example: Upload what a block computes for each user, by writing the records in the block
|
|
181
|
+
# job.upload do |records|
|
|
182
|
+
# User.where(country_code: "US").find_each do |user|
|
|
183
|
+
# records << user.email.downcase
|
|
184
|
+
# end
|
|
185
|
+
# end
|
|
227
186
|
#
|
|
228
|
-
#
|
|
229
|
-
#
|
|
187
|
+
# Example: Upload the zip_code of each document that a Mongoid query returns
|
|
188
|
+
# job.upload(User.where(state: "FL"), columns: [:zip_code])
|
|
230
189
|
#
|
|
231
|
-
# Example
|
|
190
|
+
# Example: Upload the ids 200 to 421 as one `[first_id, last_id]` range per slice, of `slice_size` ids each
|
|
232
191
|
# job.input_category.slice_size = 100
|
|
233
|
-
# job.
|
|
234
|
-
#
|
|
235
|
-
# # Equivalent to calling:
|
|
236
|
-
# job.input.insert([200,299])
|
|
237
|
-
# job.input.insert([300,399])
|
|
238
|
-
# job.input.insert([400,421])
|
|
192
|
+
# job.upload(200..421)
|
|
239
193
|
#
|
|
240
|
-
#
|
|
241
|
-
#
|
|
242
|
-
#
|
|
243
|
-
#
|
|
244
|
-
# so that if a job is retried during an upload failure, data is not duplicated.
|
|
245
|
-
|
|
246
|
-
# Upload sliced range of integer requests as an arrays of start and end ids
|
|
247
|
-
# starting with the last range first
|
|
194
|
+
# # Uploads the slices:
|
|
195
|
+
# [200, 299]
|
|
196
|
+
# [300, 399]
|
|
197
|
+
# [400, 421]
|
|
248
198
|
#
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
# Uploads one range per slice so that the response can return multiple records
|
|
252
|
-
# for each slice processed.
|
|
253
|
-
# Useful for when the highest order integer values should be processed before
|
|
254
|
-
# the lower integer value ranges. For example when processing every record
|
|
255
|
-
# in a database based on the id column
|
|
256
|
-
#
|
|
257
|
-
# Example
|
|
199
|
+
# Example: Upload the same ids from the last one, such as to process the highest ids first
|
|
258
200
|
# job.input_category.slice_size = 100
|
|
259
|
-
# job.
|
|
260
|
-
#
|
|
261
|
-
# #
|
|
262
|
-
#
|
|
263
|
-
#
|
|
264
|
-
#
|
|
265
|
-
#
|
|
266
|
-
# Notes:
|
|
267
|
-
# *
|
|
268
|
-
#
|
|
269
|
-
# *
|
|
270
|
-
#
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
201
|
+
# job.upload(421..200)
|
|
202
|
+
#
|
|
203
|
+
# # Uploads the slices:
|
|
204
|
+
# [322, 421]
|
|
205
|
+
# [222, 321]
|
|
206
|
+
# [200, 221]
|
|
207
|
+
#
|
|
208
|
+
# Notes for a query or a range:
|
|
209
|
+
# * `columns:` is only for a query: the columns, or fields, to upload. One column uploads its value, and several
|
|
210
|
+
# an Array of their values. Default: the id.
|
|
211
|
+
# * The rows of an Active Record relation are read with `find_each`, selecting the columns and the id, and each
|
|
212
|
+
# column is uploaded as the reader of the model returns it. See Sliced::Input::QueryDeprecation for what
|
|
213
|
+
# Rocket Job 8 uploads differently.
|
|
214
|
+
# * The documents of a Mongoid query are read with the MongoDB driver, without building a model for each one.
|
|
215
|
+
# * A block passed with a query is deprecated, since Rocket Job 8 raises ArgumentError for it: it receives each
|
|
216
|
+
# model of a relation, or each document of a Mongoid query as a Hash, and returns the record to upload. Supplying
|
|
217
|
+
# `columns:` with it raises ArgumentError. Write the records in the block of upload instead, as above.
|
|
218
|
+
# * The number that is returned and added to `record_count` is the number of records uploaded, which for a range
|
|
219
|
+
# is the number of `[first_id, last_id]` ranges, one per slice.
|
|
220
|
+
# * If an exception is raised while uploading data, the records that this upload stored are removed, so that
|
|
221
|
+
# if the job is retried, its data is not duplicated. The records of earlier uploads are kept.
|
|
222
|
+
def upload(object = nil, category: :main, file_name: nil, stream_mode: UPLOAD_OPTION_OMITTED, validate_rows: nil,
|
|
223
|
+
on_first: UPLOAD_OPTION_OMITTED, columns: nil, slice_batch_size: nil, **args, &block)
|
|
224
|
+
# Rocket Job 8 raises for on_first and stream_mode whenever they are passed, also as nil, whatever is uploaded.
|
|
225
|
+
if on_first.equal?(UPLOAD_OPTION_OMITTED)
|
|
226
|
+
on_first = nil
|
|
227
|
+
else
|
|
228
|
+
rocketjob_warn_on_first
|
|
229
|
+
end
|
|
274
230
|
input_collection = input(category)
|
|
231
|
+
if stream_mode.equal?(UPLOAD_OPTION_OMITTED)
|
|
232
|
+
stream_mode = nil
|
|
233
|
+
else
|
|
234
|
+
rocketjob_check_stream_mode(stream_mode, category)
|
|
235
|
+
end
|
|
275
236
|
|
|
276
|
-
|
|
237
|
+
# A block uploads the records that it writes, unless it is passed with a query, whose models or documents it
|
|
238
|
+
# transforms, as upload_arel and upload_mongo_query did. Rocket Job 8 raises for a block passed with a query, see
|
|
239
|
+
# Sliced::Input::QueryDeprecation.
|
|
240
|
+
if block && !rocketjob_query?(object)
|
|
277
241
|
raise(ArgumentError, "Cannot supply both an object to upload, and a block.") if object
|
|
242
|
+
|
|
243
|
+
rocketjob_check_no_validate_rows(validate_rows, "a block, whose records are uploaded as they are")
|
|
278
244
|
if stream_mode || columns || slice_batch_size || args.size.positive?
|
|
279
245
|
raise(ArgumentError,
|
|
280
246
|
"Unknown keyword arguments when uploading a block. Only accepts :category, :file_name, or :on_first")
|
|
@@ -284,16 +250,20 @@ module RocketJob
|
|
|
284
250
|
category.file_name = file_name if file_name
|
|
285
251
|
|
|
286
252
|
# Extract the header line during the upload when applicable.
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
253
|
+
count = category.upload_header(on_first, uploaded: input_collection.exists?) do |extract_header|
|
|
254
|
+
input_collection.upload(on_first: extract_header, slice_batch_size: slice_batch_size, &block)
|
|
255
|
+
end
|
|
290
256
|
self.record_count = (record_count || 0) + count
|
|
291
257
|
return count
|
|
292
258
|
end
|
|
293
259
|
|
|
260
|
+
# The block passed with a query returns the record to upload in place of the columns.
|
|
261
|
+
raise(ArgumentError, "Cannot supply both :columns and a block when uploading a query.") if block && columns
|
|
262
|
+
|
|
294
263
|
count =
|
|
295
264
|
case object
|
|
296
265
|
when Range
|
|
266
|
+
rocketjob_check_no_validate_rows(validate_rows, "a Range, whose values are not the lines of a file")
|
|
297
267
|
if file_name || stream_mode || on_first || args.size.positive?
|
|
298
268
|
raise(ArgumentError,
|
|
299
269
|
"Unknown keyword arguments when uploading a Range. Only accepts :category, :columns, or :slice_batch_size")
|
|
@@ -307,6 +277,7 @@ module RocketJob
|
|
|
307
277
|
input_collection.upload_integer_range_in_reverse_order(last, first, slice_batch_size: slice_batch_size || 1_000)
|
|
308
278
|
end
|
|
309
279
|
when Mongoid::Criteria
|
|
280
|
+
rocketjob_check_no_validate_rows(validate_rows, "a Mongoid::Criteria, whose values are not the lines of a file")
|
|
310
281
|
if file_name || stream_mode || on_first || args.size.positive?
|
|
311
282
|
raise(ArgumentError,
|
|
312
283
|
"Unknown keyword arguments when uploading a Mongoid::Criteria. Only accepts :category, :columns, or :slice_batch_size")
|
|
@@ -314,6 +285,7 @@ module RocketJob
|
|
|
314
285
|
|
|
315
286
|
input_collection.upload_mongo_query(object, columns: columns, slice_batch_size: slice_batch_size, &block)
|
|
316
287
|
when defined?(ActiveRecord::Relation) ? ActiveRecord::Relation : false
|
|
288
|
+
rocketjob_check_no_validate_rows(validate_rows, "an ActiveRecord::Relation, whose values are not the lines of a file")
|
|
317
289
|
if file_name || stream_mode || on_first || args.size.positive?
|
|
318
290
|
raise(ArgumentError,
|
|
319
291
|
"Unknown keyword arguments when uploading an ActiveRecord::Relation. Only accepts :category, :columns, or :slice_batch_size")
|
|
@@ -324,14 +296,21 @@ module RocketJob
|
|
|
324
296
|
else
|
|
325
297
|
raise(ArgumentError, "Unknown keyword argument :columns when uploading a file") if columns
|
|
326
298
|
|
|
299
|
+
if stream_mode && !validate_rows.nil?
|
|
300
|
+
raise(ArgumentError, "Supply either validate_rows:, or the deprecated stream_mode:, when uploading a file")
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
rocketjob_warn_deprecated_upload_options(args)
|
|
327
304
|
category = input_category(category)
|
|
305
|
+
category.check_upload(validate_rows)
|
|
328
306
|
|
|
329
|
-
#
|
|
330
|
-
|
|
307
|
+
# The path sets the format of a category whose format is :auto, which decides whether the file has a
|
|
308
|
+
# header row to extract during the upload.
|
|
331
309
|
path = category.upload_path(object, original_file_name: file_name)
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
310
|
+
category.upload_header(on_first, uploaded: input_collection.exists?) do |extract_header|
|
|
311
|
+
input_collection.upload(on_first: extract_header, slice_batch_size: slice_batch_size) do |io|
|
|
312
|
+
category.each_record(path, mode: stream_mode, validate_rows: validate_rows, **args) { |record| io << record }
|
|
313
|
+
end
|
|
335
314
|
end
|
|
336
315
|
|
|
337
316
|
end
|
|
@@ -340,29 +319,45 @@ module RocketJob
|
|
|
340
319
|
count
|
|
341
320
|
end
|
|
342
321
|
|
|
343
|
-
# @deprecated
|
|
322
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the relation instead.
|
|
344
323
|
def upload_arel(arel, *column_names, category: :main, &)
|
|
324
|
+
rocketjob_warn_removed_upload_method(
|
|
325
|
+
"upload_arel", "the relation, with the columns to upload as columns:, such as " \
|
|
326
|
+
"upload(User.where(country_code: \"US\"), columns: %i[name zip_code])"
|
|
327
|
+
)
|
|
345
328
|
count = input(category).upload_arel(arel, columns: column_names, &)
|
|
346
329
|
self.record_count = (record_count || 0) + count
|
|
347
330
|
count
|
|
348
331
|
end
|
|
349
332
|
|
|
350
|
-
# @deprecated
|
|
333
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the criteria instead.
|
|
351
334
|
def upload_mongo_query(criteria, *column_names, category: :main, &)
|
|
335
|
+
rocketjob_warn_removed_upload_method(
|
|
336
|
+
"upload_mongo_query", "the criteria, with the fields to upload as columns:, such as " \
|
|
337
|
+
"upload(User.where(state: \"FL\"), columns: [:zip_code])"
|
|
338
|
+
)
|
|
352
339
|
count = input(category).upload_mongo_query(criteria, columns: column_names, &)
|
|
353
340
|
self.record_count = (record_count || 0) + count
|
|
354
341
|
count
|
|
355
342
|
end
|
|
356
343
|
|
|
357
|
-
# @deprecated
|
|
344
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the range instead.
|
|
358
345
|
def upload_integer_range(start_id, last_id, category: :main, slice_batch_size: 1_000)
|
|
346
|
+
rocketjob_warn_removed_upload_method(
|
|
347
|
+
"upload_integer_range", "the range, such as upload(1..1_000_000) for upload_integer_range(1, 1_000_000)"
|
|
348
|
+
)
|
|
359
349
|
count = input(category).upload_integer_range(start_id, last_id, slice_batch_size: slice_batch_size)
|
|
360
350
|
self.record_count = (record_count || 0) + count
|
|
361
351
|
count
|
|
362
352
|
end
|
|
363
353
|
|
|
364
|
-
# @deprecated
|
|
354
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the range from its last id instead.
|
|
365
355
|
def upload_integer_range_in_reverse_order(start_id, last_id, category: :main, slice_batch_size: 1_000)
|
|
356
|
+
rocketjob_warn_removed_upload_method(
|
|
357
|
+
"upload_integer_range_in_reverse_order",
|
|
358
|
+
"the range from its last id, such as upload(1_000_000..1) for " \
|
|
359
|
+
"upload_integer_range_in_reverse_order(1, 1_000_000)"
|
|
360
|
+
)
|
|
366
361
|
count = input(category).upload_integer_range_in_reverse_order(start_id, last_id,
|
|
367
362
|
slice_batch_size: slice_batch_size)
|
|
368
363
|
self.record_count = (record_count || 0) + count
|
|
@@ -392,6 +387,35 @@ module RocketJob
|
|
|
392
387
|
count
|
|
393
388
|
end
|
|
394
389
|
|
|
390
|
+
# The default of the deprecated `on_first:` and `stream_mode:` options of #upload, so that passing either, even as
|
|
391
|
+
# nil, can be told from leaving it out, see #rocketjob_warn_on_first and #rocketjob_check_stream_mode. Named for
|
|
392
|
+
# upload, since a batch job includes this module, where a constant named OMITTED would hide one of the
|
|
393
|
+
# application's.
|
|
394
|
+
UPLOAD_OPTION_OMITTED = Object.new.freeze
|
|
395
|
+
private_constant :UPLOAD_OPTION_OMITTED
|
|
396
|
+
|
|
397
|
+
# The options that IOStreams reads the lines of a file with, which are deprecated, since Rocket Job 8 reads the
|
|
398
|
+
# lines itself: IOStreams detects the line endings of a file, and the format of the input category reads a quoted
|
|
399
|
+
# value that holds line breaks. Each holds what its deprecation warning says replaces it, see
|
|
400
|
+
# #rocketjob_warn_deprecated_upload_options.
|
|
401
|
+
DEPRECATED_LINE_UPLOAD_OPTIONS = {
|
|
402
|
+
delimiter: "line endings are detected, and records separated by something else are written in the " \
|
|
403
|
+
'block of upload, such as with IOStreams.path(file_name).each(:line, delimiter: "|")',
|
|
404
|
+
buffer_size: "the size of the buffer that it reads with is its own",
|
|
405
|
+
embedded_within: "the input category's format reads a quoted value with line breaks, while 7.2 reads one in " \
|
|
406
|
+
"the format of the path that is uploaded, such as a file name ending in .csv, or " \
|
|
407
|
+
"IOStreams.stream(io).format(:csv)"
|
|
408
|
+
}.freeze
|
|
409
|
+
private_constant :DEPRECATED_LINE_UPLOAD_OPTIONS
|
|
410
|
+
|
|
411
|
+
# Returns [Array<Symbol>] the options of #upload that are deprecated: the column options that the input category
|
|
412
|
+
# reads a file with, see RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS, the options that read the lines
|
|
413
|
+
# of a file, see DEPRECATED_LINE_UPLOAD_OPTIONS, `on_first`, see #rocketjob_warn_on_first, and `stream_mode`, see
|
|
414
|
+
# #rocketjob_check_stream_mode.
|
|
415
|
+
def self.deprecated_upload_options
|
|
416
|
+
[*RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS, *DEPRECATED_LINE_UPLOAD_OPTIONS.keys, :on_first, :stream_mode]
|
|
417
|
+
end
|
|
418
|
+
|
|
395
419
|
# Download the output data into the supplied file, io, IOStreams::Path, or IOStreams::Stream.
|
|
396
420
|
# Returns [Integer] the number of records / lines downloaded.
|
|
397
421
|
#
|
|
@@ -417,7 +441,7 @@ module RocketJob
|
|
|
417
441
|
# job.download(path)
|
|
418
442
|
#
|
|
419
443
|
# Example: Supply custom options. Set the file name within the zip file.
|
|
420
|
-
# path = IOStreams.path('myfile.csv.zip').option(:zip,
|
|
444
|
+
# path = IOStreams.path('myfile.csv.zip').option(:zip, entry_file_name: 'myfile.csv')
|
|
421
445
|
# job.download(path)
|
|
422
446
|
#
|
|
423
447
|
# Example: Download into a tempfile, or stream, using the original file name to determine the streams to apply:
|
|
@@ -479,6 +503,88 @@ module RocketJob
|
|
|
479
503
|
end
|
|
480
504
|
end
|
|
481
505
|
end
|
|
506
|
+
|
|
507
|
+
private
|
|
508
|
+
|
|
509
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that calls the named
|
|
510
|
+
# upload method, which Rocket Job 8 removes, saying what to call #upload with instead.
|
|
511
|
+
def rocketjob_warn_removed_upload_method(name, replacement)
|
|
512
|
+
RocketJob::Deprecation.warn(
|
|
513
|
+
"#{name} is deprecated, and removed in Rocket Job 8: call upload with #{replacement}, see " \
|
|
514
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
515
|
+
)
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# Returns [true|false] whether the supplied object is a query that #upload uploads the records of, whose models
|
|
519
|
+
# or documents a block can transform: a Mongoid::Criteria, or an ActiveRecord::Relation.
|
|
520
|
+
def rocketjob_query?(object)
|
|
521
|
+
object.is_a?(Mongoid::Criteria) || (defined?(ActiveRecord::Relation) && object.is_a?(ActiveRecord::Relation))
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads with
|
|
525
|
+
# `on_first`, also as nil, whatever it uploads, since Rocket Job 8 removes it, and raises ArgumentError for it: Rocket
|
|
526
|
+
# Job reads the header row itself, and a file without one, such as JSON or fixed width, loses its first record,
|
|
527
|
+
# which a block that writes the other lines leaves out instead.
|
|
528
|
+
def rocketjob_warn_on_first
|
|
529
|
+
RocketJob::Deprecation.warn(
|
|
530
|
+
"on_first: is deprecated, and removed in Rocket Job 8, which raises for it, also when nil: Rocket Job reads " \
|
|
531
|
+
"the header row itself, which the category's schema checks, and to leave out the first record of a file " \
|
|
532
|
+
"without a header row, upload with a block that writes the other lines, see " \
|
|
533
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
534
|
+
)
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
# Reports a deprecation warning, see RocketJob::Category::Input#warn_deprecated_stream_mode, for the supplied
|
|
538
|
+
# `stream_mode:` of an upload into the supplied category, also when nil, whatever is uploaded, since Rocket Job 8
|
|
539
|
+
# removes it, and raises ArgumentError for it. Raises ArgumentError for a stream_mode other than :line, :array or
|
|
540
|
+
# :hash, before warning for it, as 7.1 did without a schema. With a schema 7.1 read such a stream_mode as :hash.
|
|
541
|
+
def rocketjob_check_stream_mode(stream_mode, category)
|
|
542
|
+
if stream_mode && !%w[line array hash].include?(stream_mode.to_s)
|
|
543
|
+
raise(ArgumentError, "Invalid stream_mode: #{stream_mode.inspect}, which can only be :line, :array or :hash")
|
|
544
|
+
end
|
|
545
|
+
|
|
546
|
+
input_category(category).warn_deprecated_stream_mode(stream_mode)
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# Raises ArgumentError for the supplied `validate_rows:` of an upload of what is described, which has no lines to
|
|
550
|
+
# validate, unless it is nil or false, which validates nothing.
|
|
551
|
+
def rocketjob_check_no_validate_rows(validate_rows, what)
|
|
552
|
+
return if [nil, false].include?(validate_rows)
|
|
553
|
+
|
|
554
|
+
raise(ArgumentError, "validate_rows: #{validate_rows.inspect} cannot be supplied when uploading #{what}")
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads a file
|
|
558
|
+
# with options that describe the columns of the file, which the schema of the input category replaces, see
|
|
559
|
+
# RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS, and another
|
|
560
|
+
# for options that read its lines, which Rocket Job 8 removes, saying what replaces each of those that is passed,
|
|
561
|
+
# see DEPRECATED_LINE_UPLOAD_OPTIONS: IOStreams detects the line endings of a file, \r\n or \n, the size of the
|
|
562
|
+
# buffer that it reads with is its own, and the format of the input category, such as CSV, reads a quoted value
|
|
563
|
+
# that holds line breaks. 7.2 reads such a value in the format of the path that is uploaded, which has one when
|
|
564
|
+
# its file name gives it, or when one is set on the path, see Category::Input#upload_path.
|
|
565
|
+
def rocketjob_warn_deprecated_upload_options(args)
|
|
566
|
+
rocketjob_warn_upload_options(
|
|
567
|
+
args.keys & RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS,
|
|
568
|
+
"describe the columns of the file with the schema: of the input category instead, see " \
|
|
569
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE}"
|
|
570
|
+
)
|
|
571
|
+
names = args.keys & DEPRECATED_LINE_UPLOAD_OPTIONS.keys
|
|
572
|
+
rocketjob_warn_upload_options(
|
|
573
|
+
names,
|
|
574
|
+
"and removed in Rocket Job 8, which reads the lines of a file itself: " \
|
|
575
|
+
"#{DEPRECATED_LINE_UPLOAD_OPTIONS.values_at(*names).join('; ')}, see #{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
576
|
+
)
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, naming the supplied deprecated options of an upload
|
|
580
|
+
# that were passed, if any, followed by the supplied reason.
|
|
581
|
+
def rocketjob_warn_upload_options(names, reason)
|
|
582
|
+
return if names.empty?
|
|
583
|
+
|
|
584
|
+
RocketJob::Deprecation.warn(
|
|
585
|
+
"#{names.collect { |name| "#{name}:" }.join(', ')} #{names.size == 1 ? 'is' : 'are'} deprecated, #{reason}"
|
|
586
|
+
)
|
|
587
|
+
end
|
|
482
588
|
end
|
|
483
589
|
end
|
|
484
590
|
end
|