rocketjob 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rocket_job/batch/categories.rb +145 -37
- data/lib/rocket_job/batch/io.rb +222 -141
- data/lib/rocket_job/category/base.rb +57 -0
- data/lib/rocket_job/category/input.rb +252 -62
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +32 -0
- data/lib/rocket_job/deprecation.rb +12 -2
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +103 -19
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +9 -2
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +4 -0
- metadata +6 -2
data/lib/rocket_job/batch/io.rb
CHANGED
|
@@ -80,6 +80,7 @@ module RocketJob
|
|
|
80
80
|
# Or, an IOStreams path such as IOStreams::Paths::File, or IOStreams::Paths::S3
|
|
81
81
|
#
|
|
82
82
|
# delimiter[String]
|
|
83
|
+
# Deprecated, and removed in Rocket Job 8, which detects the line endings of a file.
|
|
83
84
|
# Line / Record delimiter to use to break the stream up into records
|
|
84
85
|
# Any string to break the stream up by
|
|
85
86
|
# The records when saved will not include this delimiter
|
|
@@ -88,16 +89,26 @@ module RocketJob
|
|
|
88
89
|
# Searches for the first "\r\n" or "\n" and then uses that as the
|
|
89
90
|
# delimiter for all subsequent records
|
|
90
91
|
#
|
|
92
|
+
# validate_rows: [true | false]
|
|
93
|
+
# true to parse each line of the file in the input category's format while it is uploaded, so that a file
|
|
94
|
+
# with a line that cannot be read, or with a value that is not valid, uploads nothing. The lines are still
|
|
95
|
+
# uploaded as they are, and the workers parse them.
|
|
96
|
+
# Raises ArgumentError when true for an input category without a format, or for a query, a range or a block,
|
|
97
|
+
# which upload values rather than the lines of a file, for which false is accepted, since it validates
|
|
98
|
+
# nothing. Also raises when supplied, true or false, with a `stream_mode:` other than nil or false, which
|
|
99
|
+
# decides alone whether the lines are validated.
|
|
100
|
+
# Default: The input category's `validate_rows`, see RocketJob::Category::Input#upload_validates_rows?.
|
|
101
|
+
#
|
|
91
102
|
# stream_mode: [:line | :array | :hash]
|
|
103
|
+
# Deprecated, and removed in Rocket Job 8, which raises ArgumentError whenever it is supplied, also as nil, so
|
|
104
|
+
# it reports a deprecation warning then too: `validate_rows: true` replaces :hash, and for an input category
|
|
105
|
+
# without a format, its `format: :auto` reads a file into Hashes.
|
|
92
106
|
# :line
|
|
93
107
|
# Uploads the file a line (String) at a time for processing by workers.
|
|
94
108
|
# :array
|
|
95
109
|
# Parses each line from the file as an Array and uploads each array for processing by workers.
|
|
96
110
|
# :hash
|
|
97
111
|
# Parses each line from the file into a Hash and uploads each hash for processing by workers.
|
|
98
|
-
# Default: :line
|
|
99
|
-
# With a schema, the default is the mode of the input category, and the lines of the file are uploaded
|
|
100
|
-
# in the :line and :hash modes, see RocketJob::Category::Input#mode.
|
|
101
112
|
# See IOStreams::Stream#each.
|
|
102
113
|
#
|
|
103
114
|
# category [Symbol|RocketJob::Category::Input]
|
|
@@ -143,141 +154,93 @@ module RocketJob
|
|
|
143
154
|
#
|
|
144
155
|
# Notes:
|
|
145
156
|
# * Only call from one thread at a time against a single instance of this job.
|
|
146
|
-
# * The
|
|
147
|
-
# * If an exception is raised while uploading data, the
|
|
148
|
-
#
|
|
149
|
-
# *
|
|
150
|
-
#
|
|
157
|
+
# * The number of records uploaded is added to the record_count of the job.
|
|
158
|
+
# * If an exception is raised while uploading data, the records that this upload stored are removed, so that
|
|
159
|
+
# if the job is retried, its data is not duplicated. The records of earlier uploads are kept.
|
|
160
|
+
# * The text of the file/stream is read as UTF-8 before being persisted, since MongoDB only stores UTF-8 strings.
|
|
161
|
+
# Text that is not valid UTF-8 raises IOStreams::Errors::InvalidEncoding, naming its line, unless an encoding
|
|
162
|
+
# is set on the path, such as `IOStreams.path("names.txt").encoding("ISO-8859-1")`, which reads it instead.
|
|
163
|
+
# An input category with a format but no schema ignores an encoding set on the path, and removes
|
|
164
|
+
# non-printable characters, and every character that is not ASCII, instead.
|
|
151
165
|
# * When zip format, the Zip file/stream must contain only one file, the first file found will be
|
|
152
166
|
# loaded into the job
|
|
153
167
|
# * If an io stream is supplied, it is read until it returns nil.
|
|
154
168
|
# * Only use this method for UTF-8 data, for binary data use #input_slice or #input_records.
|
|
155
169
|
# * CSV parsing is slow, so it is usually left for the workers to do.
|
|
156
170
|
#
|
|
157
|
-
# Upload
|
|
158
|
-
#
|
|
159
|
-
# Params
|
|
160
|
-
# column_names
|
|
161
|
-
# When a block is not supplied, supply the names of the columns to be returned
|
|
162
|
-
# and uploaded into the job
|
|
163
|
-
# These columns are automatically added to the select list to reduce overhead
|
|
164
|
-
#
|
|
165
|
-
# category [Symbol|RocketJob::Category::Input]
|
|
166
|
-
# The category or the name of the category to upload to.
|
|
167
|
-
#
|
|
168
|
-
# If a Block is supplied it is passed the model returned from the database and should
|
|
169
|
-
# return the work item to be uploaded into the job.
|
|
170
|
-
#
|
|
171
|
-
# Returns [Integer] the number of records uploaded
|
|
172
|
-
#
|
|
173
|
-
# Example: Upload id's for all users
|
|
174
|
-
# arel = User.all
|
|
175
|
-
# job.upload_arel(arel)
|
|
176
|
-
#
|
|
177
|
-
# Example: Upload selected user id's
|
|
178
|
-
# arel = User.where(country_code: 'US')
|
|
179
|
-
# job.upload_arel(arel)
|
|
180
|
-
#
|
|
181
|
-
# Example: Upload user_name and zip_code
|
|
182
|
-
# arel = User.where(country_code: 'US')
|
|
183
|
-
# job.upload_arel(arel, columns: [:user_name, :zip_code])
|
|
171
|
+
# Upload the records of a query or a range of integers, which replaces upload_arel, upload_mongo_query,
|
|
172
|
+
# upload_integer_range and upload_integer_range_in_reverse_order, that Rocket Job 8 removes:
|
|
184
173
|
#
|
|
185
|
-
#
|
|
186
|
-
#
|
|
187
|
-
# * The record_count for the job is set to the number of records returned by the arel.
|
|
188
|
-
# * If an exception is raised while uploading data, the input collection is cleared out
|
|
189
|
-
# so that if a job is retried during an upload failure, data is not duplicated.
|
|
190
|
-
|
|
191
|
-
# Upload the result of a MongoDB query to the input collection for processing
|
|
192
|
-
# Useful when an entire MongoDB collection, or part thereof needs to be
|
|
193
|
-
# processed by a job.
|
|
194
|
-
#
|
|
195
|
-
# Returns [Integer] the number of records uploaded
|
|
174
|
+
# Example: Upload the id of each user that an Active Record relation returns
|
|
175
|
+
# job.upload(User.where(country_code: "US"))
|
|
196
176
|
#
|
|
197
|
-
#
|
|
198
|
-
#
|
|
199
|
-
#
|
|
200
|
-
# If no Block is supplied then the record will be the :fields returned
|
|
201
|
-
# from MongoDB
|
|
202
|
-
#
|
|
203
|
-
# Note:
|
|
204
|
-
# This method uses the collection and not the MongoMapper document to
|
|
205
|
-
# avoid the overhead of constructing a Model with every document returned
|
|
206
|
-
# by the query
|
|
207
|
-
#
|
|
208
|
-
# Note:
|
|
209
|
-
# The Block must return types that can be serialized to BSON.
|
|
210
|
-
# Valid Types: Hash | Array | String | Integer | Float | Symbol | Regexp | Time
|
|
211
|
-
# Invalid: Date, etc.
|
|
177
|
+
# Example: Upload the user_name and zip_code of each user, as an Array, see `columns:`
|
|
178
|
+
# job.upload(User.where(country_code: "US"), columns: %i[user_name zip_code])
|
|
212
179
|
#
|
|
213
|
-
# Example: Upload
|
|
214
|
-
#
|
|
215
|
-
#
|
|
216
|
-
#
|
|
217
|
-
#
|
|
218
|
-
#
|
|
219
|
-
# job.record_count = job.upload_mongo_query(criteria, columns: [:zip_code])
|
|
220
|
-
#
|
|
221
|
-
# Notes:
|
|
222
|
-
# * Only call from one thread at a time against a single instance of this job.
|
|
223
|
-
# * The record_count for the job is set to the number of records returned by the monqo query.
|
|
224
|
-
# * If an exception is raised while uploading data, the input collection is cleared out
|
|
225
|
-
# so that if a job is retried during an upload failure, data is not duplicated.
|
|
226
|
-
|
|
227
|
-
# Upload sliced range of integer requests as arrays of start and end ids.
|
|
228
|
-
#
|
|
229
|
-
# Returns [Integer] the number of slices uploaded.
|
|
180
|
+
# Example: Upload what a block computes for each user, by writing the records in the block
|
|
181
|
+
# job.upload do |records|
|
|
182
|
+
# User.where(country_code: "US").find_each do |user|
|
|
183
|
+
# records << user.email.downcase
|
|
184
|
+
# end
|
|
185
|
+
# end
|
|
230
186
|
#
|
|
231
|
-
#
|
|
232
|
-
#
|
|
187
|
+
# Example: Upload the zip_code of each document that a Mongoid query returns
|
|
188
|
+
# job.upload(User.where(state: "FL"), columns: [:zip_code])
|
|
233
189
|
#
|
|
234
|
-
# Example
|
|
190
|
+
# Example: Upload the ids 200 to 421 as one `[first_id, last_id]` range per slice, of `slice_size` ids each
|
|
235
191
|
# job.input_category.slice_size = 100
|
|
236
|
-
# job.
|
|
192
|
+
# job.upload(200..421)
|
|
237
193
|
#
|
|
238
|
-
# #
|
|
239
|
-
#
|
|
240
|
-
#
|
|
241
|
-
#
|
|
194
|
+
# # Uploads the slices:
|
|
195
|
+
# [200, 299]
|
|
196
|
+
# [300, 399]
|
|
197
|
+
# [400, 421]
|
|
242
198
|
#
|
|
243
|
-
#
|
|
244
|
-
# * Only call from one thread at a time against a single instance of this job.
|
|
245
|
-
# * The record_count for the job is set to: last_id - start_id + 1.
|
|
246
|
-
# * If an exception is raised while uploading data, the input collection is cleared out
|
|
247
|
-
# so that if a job is retried during an upload failure, data is not duplicated.
|
|
248
|
-
|
|
249
|
-
# Upload sliced range of integer requests as an arrays of start and end ids
|
|
250
|
-
# starting with the last range first
|
|
251
|
-
#
|
|
252
|
-
# Returns [Integer] the number of slices uploaded.
|
|
253
|
-
#
|
|
254
|
-
# Uploads one range per slice so that the response can return multiple records
|
|
255
|
-
# for each slice processed.
|
|
256
|
-
# Useful for when the highest order integer values should be processed before
|
|
257
|
-
# the lower integer value ranges. For example when processing every record
|
|
258
|
-
# in a database based on the id column
|
|
259
|
-
#
|
|
260
|
-
# Example
|
|
199
|
+
# Example: Upload the same ids from the last one, such as to process the highest ids first
|
|
261
200
|
# job.input_category.slice_size = 100
|
|
262
|
-
# job.
|
|
263
|
-
#
|
|
264
|
-
# #
|
|
265
|
-
#
|
|
266
|
-
#
|
|
267
|
-
#
|
|
268
|
-
#
|
|
269
|
-
# Notes:
|
|
270
|
-
# *
|
|
271
|
-
#
|
|
272
|
-
# *
|
|
273
|
-
#
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
201
|
+
# job.upload(421..200)
|
|
202
|
+
#
|
|
203
|
+
# # Uploads the slices:
|
|
204
|
+
# [322, 421]
|
|
205
|
+
# [222, 321]
|
|
206
|
+
# [200, 221]
|
|
207
|
+
#
|
|
208
|
+
# Notes for a query or a range:
|
|
209
|
+
# * `columns:` is only for a query: the columns, or fields, to upload. One column uploads its value, and several
|
|
210
|
+
# an Array of their values. Default: the id.
|
|
211
|
+
# * The rows of an Active Record relation are read with `find_each`, selecting the columns and the id, and each
|
|
212
|
+
# column is uploaded as the reader of the model returns it. See Sliced::Input::QueryDeprecation for what
|
|
213
|
+
# Rocket Job 8 uploads differently.
|
|
214
|
+
# * The documents of a Mongoid query are read with the MongoDB driver, without building a model for each one.
|
|
215
|
+
# * A block passed with a query is deprecated, since Rocket Job 8 raises ArgumentError for it: it receives each
|
|
216
|
+
# model of a relation, or each document of a Mongoid query as a Hash, and returns the record to upload. Supplying
|
|
217
|
+
# `columns:` with it raises ArgumentError. Write the records in the block of upload instead, as above.
|
|
218
|
+
# * The number that is returned and added to `record_count` is the number of records uploaded, which for a range
|
|
219
|
+
# is the number of `[first_id, last_id]` ranges, one per slice.
|
|
220
|
+
# * If an exception is raised while uploading data, the records that this upload stored are removed, so that
|
|
221
|
+
# if the job is retried, its data is not duplicated. The records of earlier uploads are kept.
|
|
222
|
+
def upload(object = nil, category: :main, file_name: nil, stream_mode: UPLOAD_OPTION_OMITTED, validate_rows: nil,
|
|
223
|
+
on_first: UPLOAD_OPTION_OMITTED, columns: nil, slice_batch_size: nil, **args, &block)
|
|
224
|
+
# Rocket Job 8 raises for on_first and stream_mode whenever they are passed, also as nil, whatever is uploaded.
|
|
225
|
+
if on_first.equal?(UPLOAD_OPTION_OMITTED)
|
|
226
|
+
on_first = nil
|
|
227
|
+
else
|
|
228
|
+
rocketjob_warn_on_first
|
|
229
|
+
end
|
|
277
230
|
input_collection = input(category)
|
|
231
|
+
if stream_mode.equal?(UPLOAD_OPTION_OMITTED)
|
|
232
|
+
stream_mode = nil
|
|
233
|
+
else
|
|
234
|
+
rocketjob_check_stream_mode(stream_mode, category)
|
|
235
|
+
end
|
|
278
236
|
|
|
279
|
-
|
|
237
|
+
# A block uploads the records that it writes, unless it is passed with a query, whose models or documents it
|
|
238
|
+
# transforms, as upload_arel and upload_mongo_query did. Rocket Job 8 raises for a block passed with a query, see
|
|
239
|
+
# Sliced::Input::QueryDeprecation.
|
|
240
|
+
if block && !rocketjob_query?(object)
|
|
280
241
|
raise(ArgumentError, "Cannot supply both an object to upload, and a block.") if object
|
|
242
|
+
|
|
243
|
+
rocketjob_check_no_validate_rows(validate_rows, "a block, whose records are uploaded as they are")
|
|
281
244
|
if stream_mode || columns || slice_batch_size || args.size.positive?
|
|
282
245
|
raise(ArgumentError,
|
|
283
246
|
"Unknown keyword arguments when uploading a block. Only accepts :category, :file_name, or :on_first")
|
|
@@ -287,16 +250,20 @@ module RocketJob
|
|
|
287
250
|
category.file_name = file_name if file_name
|
|
288
251
|
|
|
289
252
|
# Extract the header line during the upload when applicable.
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
253
|
+
count = category.upload_header(on_first, uploaded: input_collection.exists?) do |extract_header|
|
|
254
|
+
input_collection.upload(on_first: extract_header, slice_batch_size: slice_batch_size, &block)
|
|
255
|
+
end
|
|
293
256
|
self.record_count = (record_count || 0) + count
|
|
294
257
|
return count
|
|
295
258
|
end
|
|
296
259
|
|
|
260
|
+
# The block passed with a query returns the record to upload in place of the columns.
|
|
261
|
+
raise(ArgumentError, "Cannot supply both :columns and a block when uploading a query.") if block && columns
|
|
262
|
+
|
|
297
263
|
count =
|
|
298
264
|
case object
|
|
299
265
|
when Range
|
|
266
|
+
rocketjob_check_no_validate_rows(validate_rows, "a Range, whose values are not the lines of a file")
|
|
300
267
|
if file_name || stream_mode || on_first || args.size.positive?
|
|
301
268
|
raise(ArgumentError,
|
|
302
269
|
"Unknown keyword arguments when uploading a Range. Only accepts :category, :columns, or :slice_batch_size")
|
|
@@ -310,6 +277,7 @@ module RocketJob
|
|
|
310
277
|
input_collection.upload_integer_range_in_reverse_order(last, first, slice_batch_size: slice_batch_size || 1_000)
|
|
311
278
|
end
|
|
312
279
|
when Mongoid::Criteria
|
|
280
|
+
rocketjob_check_no_validate_rows(validate_rows, "a Mongoid::Criteria, whose values are not the lines of a file")
|
|
313
281
|
if file_name || stream_mode || on_first || args.size.positive?
|
|
314
282
|
raise(ArgumentError,
|
|
315
283
|
"Unknown keyword arguments when uploading a Mongoid::Criteria. Only accepts :category, :columns, or :slice_batch_size")
|
|
@@ -317,6 +285,7 @@ module RocketJob
|
|
|
317
285
|
|
|
318
286
|
input_collection.upload_mongo_query(object, columns: columns, slice_batch_size: slice_batch_size, &block)
|
|
319
287
|
when defined?(ActiveRecord::Relation) ? ActiveRecord::Relation : false
|
|
288
|
+
rocketjob_check_no_validate_rows(validate_rows, "an ActiveRecord::Relation, whose values are not the lines of a file")
|
|
320
289
|
if file_name || stream_mode || on_first || args.size.positive?
|
|
321
290
|
raise(ArgumentError,
|
|
322
291
|
"Unknown keyword arguments when uploading an ActiveRecord::Relation. Only accepts :category, :columns, or :slice_batch_size")
|
|
@@ -327,16 +296,21 @@ module RocketJob
|
|
|
327
296
|
else
|
|
328
297
|
raise(ArgumentError, "Unknown keyword argument :columns when uploading a file") if columns
|
|
329
298
|
|
|
299
|
+
if stream_mode && !validate_rows.nil?
|
|
300
|
+
raise(ArgumentError, "Supply either validate_rows:, or the deprecated stream_mode:, when uploading a file")
|
|
301
|
+
end
|
|
302
|
+
|
|
330
303
|
rocketjob_warn_deprecated_upload_options(args)
|
|
331
304
|
category = input_category(category)
|
|
305
|
+
category.check_upload(validate_rows)
|
|
332
306
|
|
|
333
307
|
# The path sets the format of a category whose format is :auto, which decides whether the file has a
|
|
334
308
|
# header row to extract during the upload.
|
|
335
|
-
path
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
309
|
+
path = category.upload_path(object, original_file_name: file_name)
|
|
310
|
+
category.upload_header(on_first, uploaded: input_collection.exists?) do |extract_header|
|
|
311
|
+
input_collection.upload(on_first: extract_header, slice_batch_size: slice_batch_size) do |io|
|
|
312
|
+
category.each_record(path, mode: stream_mode, validate_rows: validate_rows, **args) { |record| io << record }
|
|
313
|
+
end
|
|
340
314
|
end
|
|
341
315
|
|
|
342
316
|
end
|
|
@@ -345,29 +319,45 @@ module RocketJob
|
|
|
345
319
|
count
|
|
346
320
|
end
|
|
347
321
|
|
|
348
|
-
# @deprecated
|
|
322
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the relation instead.
|
|
349
323
|
def upload_arel(arel, *column_names, category: :main, &)
|
|
324
|
+
rocketjob_warn_removed_upload_method(
|
|
325
|
+
"upload_arel", "the relation, with the columns to upload as columns:, such as " \
|
|
326
|
+
"upload(User.where(country_code: \"US\"), columns: %i[name zip_code])"
|
|
327
|
+
)
|
|
350
328
|
count = input(category).upload_arel(arel, columns: column_names, &)
|
|
351
329
|
self.record_count = (record_count || 0) + count
|
|
352
330
|
count
|
|
353
331
|
end
|
|
354
332
|
|
|
355
|
-
# @deprecated
|
|
333
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the criteria instead.
|
|
356
334
|
def upload_mongo_query(criteria, *column_names, category: :main, &)
|
|
335
|
+
rocketjob_warn_removed_upload_method(
|
|
336
|
+
"upload_mongo_query", "the criteria, with the fields to upload as columns:, such as " \
|
|
337
|
+
"upload(User.where(state: \"FL\"), columns: [:zip_code])"
|
|
338
|
+
)
|
|
357
339
|
count = input(category).upload_mongo_query(criteria, columns: column_names, &)
|
|
358
340
|
self.record_count = (record_count || 0) + count
|
|
359
341
|
count
|
|
360
342
|
end
|
|
361
343
|
|
|
362
|
-
# @deprecated
|
|
344
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the range instead.
|
|
363
345
|
def upload_integer_range(start_id, last_id, category: :main, slice_batch_size: 1_000)
|
|
346
|
+
rocketjob_warn_removed_upload_method(
|
|
347
|
+
"upload_integer_range", "the range, such as upload(1..1_000_000) for upload_integer_range(1, 1_000_000)"
|
|
348
|
+
)
|
|
364
349
|
count = input(category).upload_integer_range(start_id, last_id, slice_batch_size: slice_batch_size)
|
|
365
350
|
self.record_count = (record_count || 0) + count
|
|
366
351
|
count
|
|
367
352
|
end
|
|
368
353
|
|
|
369
|
-
# @deprecated
|
|
354
|
+
# @deprecated Removed in Rocket Job 8: call #upload with the range from its last id instead.
|
|
370
355
|
def upload_integer_range_in_reverse_order(start_id, last_id, category: :main, slice_batch_size: 1_000)
|
|
356
|
+
rocketjob_warn_removed_upload_method(
|
|
357
|
+
"upload_integer_range_in_reverse_order",
|
|
358
|
+
"the range from its last id, such as upload(1_000_000..1) for " \
|
|
359
|
+
"upload_integer_range_in_reverse_order(1, 1_000_000)"
|
|
360
|
+
)
|
|
371
361
|
count = input(category).upload_integer_range_in_reverse_order(start_id, last_id,
|
|
372
362
|
slice_batch_size: slice_batch_size)
|
|
373
363
|
self.record_count = (record_count || 0) + count
|
|
@@ -397,10 +387,34 @@ module RocketJob
|
|
|
397
387
|
count
|
|
398
388
|
end
|
|
399
389
|
|
|
400
|
-
# The
|
|
401
|
-
#
|
|
402
|
-
|
|
403
|
-
|
|
390
|
+
# The default of the deprecated `on_first:` and `stream_mode:` options of #upload, so that passing either, even as
|
|
391
|
+
# nil, can be told from leaving it out, see #rocketjob_warn_on_first and #rocketjob_check_stream_mode. Named for
|
|
392
|
+
# upload, since a batch job includes this module, where a constant named OMITTED would hide one of the
|
|
393
|
+
# application's.
|
|
394
|
+
UPLOAD_OPTION_OMITTED = Object.new.freeze
|
|
395
|
+
private_constant :UPLOAD_OPTION_OMITTED
|
|
396
|
+
|
|
397
|
+
# The options that IOStreams reads the lines of a file with, which are deprecated, since Rocket Job 8 reads the
|
|
398
|
+
# lines itself: IOStreams detects the line endings of a file, and the format of the input category reads a quoted
|
|
399
|
+
# value that holds line breaks. Each holds what its deprecation warning says replaces it, see
|
|
400
|
+
# #rocketjob_warn_deprecated_upload_options.
|
|
401
|
+
DEPRECATED_LINE_UPLOAD_OPTIONS = {
|
|
402
|
+
delimiter: "line endings are detected, and records separated by something else are written in the " \
|
|
403
|
+
'block of upload, such as with IOStreams.path(file_name).each(:line, delimiter: "|")',
|
|
404
|
+
buffer_size: "the size of the buffer that it reads with is its own",
|
|
405
|
+
embedded_within: "the input category's format reads a quoted value with line breaks, while 7.2 reads one in " \
|
|
406
|
+
"the format of the path that is uploaded, such as a file name ending in .csv, or " \
|
|
407
|
+
"IOStreams.stream(io).format(:csv)"
|
|
408
|
+
}.freeze
|
|
409
|
+
private_constant :DEPRECATED_LINE_UPLOAD_OPTIONS
|
|
410
|
+
|
|
411
|
+
# Returns [Array<Symbol>] the options of #upload that are deprecated: the column options that the input category
|
|
412
|
+
# reads a file with, see RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS, the options that read the lines
|
|
413
|
+
# of a file, see DEPRECATED_LINE_UPLOAD_OPTIONS, `on_first`, see #rocketjob_warn_on_first, and `stream_mode`, see
|
|
414
|
+
# #rocketjob_check_stream_mode.
|
|
415
|
+
def self.deprecated_upload_options
|
|
416
|
+
[*RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS, *DEPRECATED_LINE_UPLOAD_OPTIONS.keys, :on_first, :stream_mode]
|
|
417
|
+
end
|
|
404
418
|
|
|
405
419
|
# Download the output data into the supplied file, io, IOStreams::Path, or IOStreams::Stream.
|
|
406
420
|
# Returns [Integer] the number of records / lines downloaded.
|
|
@@ -492,16 +506,83 @@ module RocketJob
|
|
|
492
506
|
|
|
493
507
|
private
|
|
494
508
|
|
|
509
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that calls the named
|
|
510
|
+
# upload method, which Rocket Job 8 removes, saying what to call #upload with instead.
|
|
511
|
+
def rocketjob_warn_removed_upload_method(name, replacement)
|
|
512
|
+
RocketJob::Deprecation.warn(
|
|
513
|
+
"#{name} is deprecated, and removed in Rocket Job 8: call upload with #{replacement}, see " \
|
|
514
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
515
|
+
)
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# Returns [true|false] whether the supplied object is a query that #upload uploads the records of, whose models
|
|
519
|
+
# or documents a block can transform: a Mongoid::Criteria, or an ActiveRecord::Relation.
|
|
520
|
+
def rocketjob_query?(object)
|
|
521
|
+
object.is_a?(Mongoid::Criteria) || (defined?(ActiveRecord::Relation) && object.is_a?(ActiveRecord::Relation))
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads with
|
|
525
|
+
# `on_first`, also as nil, whatever it uploads, since Rocket Job 8 removes it, and raises ArgumentError for it: Rocket
|
|
526
|
+
# Job reads the header row itself, and a file without one, such as JSON or fixed width, loses its first record,
|
|
527
|
+
# which a block that writes the other lines leaves out instead.
|
|
528
|
+
def rocketjob_warn_on_first
|
|
529
|
+
RocketJob::Deprecation.warn(
|
|
530
|
+
"on_first: is deprecated, and removed in Rocket Job 8, which raises for it, also when nil: Rocket Job reads " \
|
|
531
|
+
"the header row itself, which the category's schema checks, and to leave out the first record of a file " \
|
|
532
|
+
"without a header row, upload with a block that writes the other lines, see " \
|
|
533
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
534
|
+
)
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
# Reports a deprecation warning, see RocketJob::Category::Input#warn_deprecated_stream_mode, for the supplied
|
|
538
|
+
# `stream_mode:` of an upload into the supplied category, also when nil, whatever is uploaded, since Rocket Job 8
|
|
539
|
+
# removes it, and raises ArgumentError for it. Raises ArgumentError for a stream_mode other than :line, :array or
|
|
540
|
+
# :hash, before warning for it, as 7.1 did without a schema. With a schema 7.1 read such a stream_mode as :hash.
|
|
541
|
+
def rocketjob_check_stream_mode(stream_mode, category)
|
|
542
|
+
if stream_mode && !%w[line array hash].include?(stream_mode.to_s)
|
|
543
|
+
raise(ArgumentError, "Invalid stream_mode: #{stream_mode.inspect}, which can only be :line, :array or :hash")
|
|
544
|
+
end
|
|
545
|
+
|
|
546
|
+
input_category(category).warn_deprecated_stream_mode(stream_mode)
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# Raises ArgumentError for the supplied `validate_rows:` of an upload of what is described, which has no lines to
|
|
550
|
+
# validate, unless it is nil or false, which validates nothing.
|
|
551
|
+
def rocketjob_check_no_validate_rows(validate_rows, what)
|
|
552
|
+
return if [nil, false].include?(validate_rows)
|
|
553
|
+
|
|
554
|
+
raise(ArgumentError, "validate_rows: #{validate_rows.inspect} cannot be supplied when uploading #{what}")
|
|
555
|
+
end
|
|
556
|
+
|
|
495
557
|
# Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads a file
|
|
496
|
-
# with options that describe the columns of the file, which the schema of the input category replaces
|
|
558
|
+
# with options that describe the columns of the file, which the schema of the input category replaces, see
|
|
559
|
+
# RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS, and another
|
|
560
|
+
# for options that read its lines, which Rocket Job 8 removes, saying what replaces each of those that is passed,
|
|
561
|
+
# see DEPRECATED_LINE_UPLOAD_OPTIONS: IOStreams detects the line endings of a file, \r\n or \n, the size of the
|
|
562
|
+
# buffer that it reads with is its own, and the format of the input category, such as CSV, reads a quoted value
|
|
563
|
+
# that holds line breaks. 7.2 reads such a value in the format of the path that is uploaded, which has one when
|
|
564
|
+
# its file name gives it, or when one is set on the path, see Category::Input#upload_path.
|
|
497
565
|
def rocketjob_warn_deprecated_upload_options(args)
|
|
498
|
-
|
|
566
|
+
rocketjob_warn_upload_options(
|
|
567
|
+
args.keys & RocketJob::Category::Input::DEPRECATED_UPLOAD_OPTIONS,
|
|
568
|
+
"describe the columns of the file with the schema: of the input category instead, see " \
|
|
569
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE}"
|
|
570
|
+
)
|
|
571
|
+
names = args.keys & DEPRECATED_LINE_UPLOAD_OPTIONS.keys
|
|
572
|
+
rocketjob_warn_upload_options(
|
|
573
|
+
names,
|
|
574
|
+
"and removed in Rocket Job 8, which reads the lines of a file itself: " \
|
|
575
|
+
"#{DEPRECATED_LINE_UPLOAD_OPTIONS.values_at(*names).join('; ')}, see #{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
576
|
+
)
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, naming the supplied deprecated options of an upload
|
|
580
|
+
# that were passed, if any, followed by the supplied reason.
|
|
581
|
+
def rocketjob_warn_upload_options(names, reason)
|
|
499
582
|
return if names.empty?
|
|
500
583
|
|
|
501
584
|
RocketJob::Deprecation.warn(
|
|
502
|
-
"#{names.collect { |name| "#{name}:" }.join(', ')} #{names.size == 1 ? 'is' : 'are'} deprecated,
|
|
503
|
-
"the columns of the file with the schema: of the input category instead, see " \
|
|
504
|
-
"#{RocketJob::Deprecation::UPGRADE_GUIDE}"
|
|
585
|
+
"#{names.collect { |name| "#{name}:" }.join(', ')} #{names.size == 1 ? 'is' : 'are'} deprecated, #{reason}"
|
|
505
586
|
)
|
|
506
587
|
end
|
|
507
588
|
end
|
|
@@ -81,6 +81,22 @@ module RocketJob
|
|
|
81
81
|
properties["columns"].present? ? ["columns"] : []
|
|
82
82
|
end
|
|
83
83
|
|
|
84
|
+
# Returns [Array<String>] the names of the deprecated options of a category, see .deprecated_options, to look for
|
|
85
|
+
# where they cannot be read as properties, such as in the code of a RocketJob::Jobs::OnDemandBatchJob. Without
|
|
86
|
+
# `columns`, which a schema still uses. Overridden by the input category.
|
|
87
|
+
def deprecated_option_names
|
|
88
|
+
%w[format_options]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Returns [Array<String>] the names of all the deprecated options in the supplied properties of a category,
|
|
92
|
+
# with String keys: its deprecated column options, see .deprecated_column_options, and `format_options`.
|
|
93
|
+
# Overridden by the input category.
|
|
94
|
+
def deprecated_options(properties)
|
|
95
|
+
names = deprecated_column_options(properties)
|
|
96
|
+
names << "format_options" if properties["format_options"].present?
|
|
97
|
+
names
|
|
98
|
+
end
|
|
99
|
+
|
|
84
100
|
# Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
|
|
85
101
|
# properties of a category, such as those that a job class defines it with, or that a Dirmon entry holds.
|
|
86
102
|
def warn_deprecated_options(properties)
|
|
@@ -100,6 +116,16 @@ module RocketJob
|
|
|
100
116
|
"file, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
|
|
101
117
|
)
|
|
102
118
|
end
|
|
119
|
+
|
|
120
|
+
# Returns [Hash] the supplied attributes or properties of a category to show to people, with its file name, and
|
|
121
|
+
# its schema when that is the path or url of a JSON file, shown by their display names, without any
|
|
122
|
+
# credentials, see RocketJob.path_display_name.
|
|
123
|
+
def display_properties(properties)
|
|
124
|
+
properties.to_h do |key, value|
|
|
125
|
+
path = key.to_s == "file_name" || (key.to_s == "schema" && value.is_a?(String))
|
|
126
|
+
[key, path ? RocketJob.path_display_name(value) : value]
|
|
127
|
+
end
|
|
128
|
+
end
|
|
103
129
|
end
|
|
104
130
|
|
|
105
131
|
# Return which slice serializer class to use that matches the current options.
|
|
@@ -142,8 +168,39 @@ module RocketJob
|
|
|
142
168
|
collection_name
|
|
143
169
|
end
|
|
144
170
|
|
|
171
|
+
# Returns [Array<String>] the names of the deprecated options that this category sets, see .deprecated_options,
|
|
172
|
+
# which #schema_proposal replaces.
|
|
173
|
+
def deprecated_options_set
|
|
174
|
+
self.class.deprecated_options(attributes.to_h.stringify_keys)
|
|
175
|
+
end
|
|
176
|
+
|
|
145
177
|
private
|
|
146
178
|
|
|
179
|
+
# Returns [Hash] the supplied deprecated options, by name, set to their defaults, as a schema proposal sets them,
|
|
180
|
+
# see RocketJob::SchemaProposal#changes.
|
|
181
|
+
def default_settings(names)
|
|
182
|
+
names.to_h { |name| [name, self.class.fields[name].default_val] }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Returns [Array] the schema of this category's fixed width layout, see #format_options, with String keys, as
|
|
186
|
+
# MongoDB stores it, and nil, or nil and why IOStreams cannot translate the layout, such as two columns with one
|
|
187
|
+
# key. The fields that the supplied columns leave out are skipped, which holds their place in each line.
|
|
188
|
+
def layout_schema(keep = nil)
|
|
189
|
+
layout = format_options.to_h.deep_symbolize_keys
|
|
190
|
+
schema = JSON.parse(JSON.generate(IOStreams::Schema.from_layout(**layout)))
|
|
191
|
+
if keep.present?
|
|
192
|
+
keep = keep.map(&:to_s)
|
|
193
|
+
schema["fields"] = schema["fields"].map do |field|
|
|
194
|
+
keep.include?(field["name"].to_s) ? field : field.slice("name", "width", "start").merge("skip" => true)
|
|
195
|
+
end
|
|
196
|
+
# 7.0 keyed the records of a layout narrowed to its columns by Strings, whatever the keys of the layout.
|
|
197
|
+
schema.delete("keyType")
|
|
198
|
+
end
|
|
199
|
+
[schema, nil]
|
|
200
|
+
rescue IOStreams::Errors::InvalidLayout, ArgumentError => e
|
|
201
|
+
[nil, "#{e.message} Write its schema by hand, see \"Moving a layout to a schema\" in IOStreams' upgrade guide"]
|
|
202
|
+
end
|
|
203
|
+
|
|
147
204
|
# Returns [IOStreams::Tabular] that reads or writes the records of this category by its schema, see #schema,
|
|
148
205
|
# reading the file that holds it when it is a path or url.
|
|
149
206
|
def schema_tabular
|