rocketjob 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/rocket_job/batch/categories.rb +161 -40
- data/lib/rocket_job/batch/io.rb +239 -133
- data/lib/rocket_job/category/base.rb +176 -1
- data/lib/rocket_job/category/input.rb +357 -41
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +39 -0
- data/lib/rocket_job/deprecation.rb +61 -0
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +114 -20
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/plugins/retry.rb +35 -34
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +13 -6
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +5 -0
- metadata +9 -4
|
@@ -15,6 +15,10 @@ module RocketJob
|
|
|
15
15
|
# The header columns when the file does not include a header row.
|
|
16
16
|
# Note:
|
|
17
17
|
# - All column names must be strings so that it can be serialized into MongoDB.
|
|
18
|
+
# - With a schema, see #schema, the columns of an input category hold the header row read from the uploaded
|
|
19
|
+
# file, see RocketJob::Category::Input, and those of an output category cannot be set.
|
|
20
|
+
# - Deprecated without a schema, use `schema`, whose fields are the columns, with `"header": false` for a file
|
|
21
|
+
# without a header row.
|
|
18
22
|
field :columns, type: Array
|
|
19
23
|
|
|
20
24
|
# On an input collection `format` specifies the format of the input data so that it can be
|
|
@@ -28,17 +32,102 @@ module RocketJob
|
|
|
28
32
|
#
|
|
29
33
|
# Default: `nil`
|
|
30
34
|
field :format, type: ::Mongoid::StringifiedSymbol
|
|
31
|
-
validates_inclusion_of :format, in: [nil, :auto] + IOStreams
|
|
35
|
+
validates_inclusion_of :format, in: [nil, :auto] + IOStreams.tabular_formats
|
|
32
36
|
|
|
33
37
|
# Any specialized format specific options. For example, `:fixed` format requires a `:layout`.
|
|
38
|
+
#
|
|
39
|
+
# Deprecated, use `schema`, see #schema, which gives the width of each field of a fixed width file. It cannot be
|
|
40
|
+
# set with a schema.
|
|
34
41
|
field :format_options, type: Hash
|
|
35
42
|
|
|
43
|
+
# The fields of the records in this category and the type of each one, as a Frictionless Table Schema, see
|
|
44
|
+
# https://datapackage.org/standard/table-schema/, so that records are read as typed values, such as an Integer,
|
|
45
|
+
# a BigDecimal or a Date, and written in the text form of each type, for every tabular format. It also gives
|
|
46
|
+
# the position of each field of a fixed width file, with its `width`, and `"header": false` describes a file
|
|
47
|
+
# without a header row. See `IOStreams::Schema`, and `IOStreams::Stream#schema`.
|
|
48
|
+
#
|
|
49
|
+
# A Hash, or the path or url of a JSON file that holds the schema, which is read when the category is first
|
|
50
|
+
# used to read or write records.
|
|
51
|
+
#
|
|
52
|
+
# Example, for a fixed width file:
|
|
53
|
+
# {"fields" => [{"name" => "name", "type" => "string", "width" => 10},
|
|
54
|
+
# {"name" => "zip", "type" => "integer", "width" => 5}]}
|
|
55
|
+
#
|
|
56
|
+
# The schema describes the columns of the file, so `format_options` and `columns` cannot be set with it, nor on
|
|
57
|
+
# an input category `allowed_columns`, `required_columns`, `skip_unknown` and `header_cleanser`. It needs a
|
|
58
|
+
# `format`, such as `:csv`, or `:auto` to take it from the file name.
|
|
59
|
+
#
|
|
60
|
+
# Default: nil, which reads and writes each value as text, as before.
|
|
61
|
+
field :schema
|
|
62
|
+
validate :schema_valid
|
|
63
|
+
validate :schema_replaces_fields
|
|
64
|
+
|
|
65
|
+
# Setting the schema builds the tabular again from it when it is next used, see #tabular.
|
|
66
|
+
define_method(:schema=) do |value|
|
|
67
|
+
@tabular = nil
|
|
68
|
+
super(value)
|
|
69
|
+
end
|
|
70
|
+
|
|
36
71
|
# When `:format` is not supplied the file name can be used to infer the required format.
|
|
37
72
|
# Optional.
|
|
38
73
|
# Default: nil
|
|
39
74
|
field :file_name, type: IOStreams::Path
|
|
40
75
|
end
|
|
41
76
|
|
|
77
|
+
module ClassMethods
|
|
78
|
+
# Returns [Array<String>] the names of the deprecated options in the supplied properties of a category, with
|
|
79
|
+
# String keys, that its schema replaces, see #schema: its `columns`. Overridden by the input category.
|
|
80
|
+
def deprecated_column_options(properties)
|
|
81
|
+
properties["columns"].present? ? ["columns"] : []
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Returns [Array<String>] the names of the deprecated options of a category, see .deprecated_options, to look for
|
|
85
|
+
# where they cannot be read as properties, such as in the code of a RocketJob::Jobs::OnDemandBatchJob. Without
|
|
86
|
+
# `columns`, which a schema still uses. Overridden by the input category.
|
|
87
|
+
def deprecated_option_names
|
|
88
|
+
%w[format_options]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Returns [Array<String>] the names of all the deprecated options in the supplied properties of a category,
|
|
92
|
+
# with String keys: its deprecated column options, see .deprecated_column_options, and `format_options`.
|
|
93
|
+
# Overridden by the input category.
|
|
94
|
+
def deprecated_options(properties)
|
|
95
|
+
names = deprecated_column_options(properties)
|
|
96
|
+
names << "format_options" if properties["format_options"].present?
|
|
97
|
+
names
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
|
|
101
|
+
# properties of a category, such as those that a job class defines it with, or that a Dirmon entry holds.
|
|
102
|
+
def warn_deprecated_options(properties)
|
|
103
|
+
properties = properties.to_h.stringify_keys
|
|
104
|
+
names = deprecated_column_options(properties)
|
|
105
|
+
unless names.empty?
|
|
106
|
+
options = names.collect { |name| "#{name}:" }.join(", ")
|
|
107
|
+
RocketJob::Deprecation.warn(
|
|
108
|
+
"#{options} #{names.size == 1 ? 'is' : 'are'} deprecated, describe the columns of the file with schema: " \
|
|
109
|
+
"instead, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
|
|
110
|
+
)
|
|
111
|
+
end
|
|
112
|
+
return if properties["format_options"].blank?
|
|
113
|
+
|
|
114
|
+
RocketJob::Deprecation.warn(
|
|
115
|
+
"format_options: is deprecated, use schema: instead, which gives the width of each field of a fixed width " \
|
|
116
|
+
"file, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
|
|
117
|
+
)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Returns [Hash] the supplied attributes or properties of a category to show to people, with its file name, and
|
|
121
|
+
# its schema when that is the path or url of a JSON file, shown by their display names, without any
|
|
122
|
+
# credentials, see RocketJob.path_display_name.
|
|
123
|
+
def display_properties(properties)
|
|
124
|
+
properties.to_h do |key, value|
|
|
125
|
+
path = key.to_s == "file_name" || (key.to_s == "schema" && value.is_a?(String))
|
|
126
|
+
[key, path ? RocketJob.path_display_name(value) : value]
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
42
131
|
# Return which slice serializer class to use that matches the current options.
|
|
43
132
|
def serializer_class
|
|
44
133
|
case serializer
|
|
@@ -58,6 +147,8 @@ module RocketJob
|
|
|
58
147
|
end
|
|
59
148
|
|
|
60
149
|
def tabular
|
|
150
|
+
return @tabular ||= schema_tabular if schema.present?
|
|
151
|
+
|
|
61
152
|
@tabular ||= IOStreams::Tabular.new(
|
|
62
153
|
columns: columns,
|
|
63
154
|
format: format == :auto ? nil : format,
|
|
@@ -76,6 +167,90 @@ module RocketJob
|
|
|
76
167
|
collection_name << ".#{name}" unless name == :main
|
|
77
168
|
collection_name
|
|
78
169
|
end
|
|
170
|
+
|
|
171
|
+
# Returns [Array<String>] the names of the deprecated options that this category sets, see .deprecated_options,
|
|
172
|
+
# which #schema_proposal replaces.
|
|
173
|
+
def deprecated_options_set
|
|
174
|
+
self.class.deprecated_options(attributes.to_h.stringify_keys)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
private
|
|
178
|
+
|
|
179
|
+
# Returns [Hash] the supplied deprecated options, by name, set to their defaults, as a schema proposal sets them,
|
|
180
|
+
# see RocketJob::SchemaProposal#changes.
|
|
181
|
+
def default_settings(names)
|
|
182
|
+
names.to_h { |name| [name, self.class.fields[name].default_val] }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Returns [Array] the schema of this category's fixed width layout, see #format_options, with String keys, as
|
|
186
|
+
# MongoDB stores it, and nil, or nil and why IOStreams cannot translate the layout, such as two columns with one
|
|
187
|
+
# key. The fields that the supplied columns leave out are skipped, which holds their place in each line.
|
|
188
|
+
def layout_schema(keep = nil)
|
|
189
|
+
layout = format_options.to_h.deep_symbolize_keys
|
|
190
|
+
schema = JSON.parse(JSON.generate(IOStreams::Schema.from_layout(**layout)))
|
|
191
|
+
if keep.present?
|
|
192
|
+
keep = keep.map(&:to_s)
|
|
193
|
+
schema["fields"] = schema["fields"].map do |field|
|
|
194
|
+
keep.include?(field["name"].to_s) ? field : field.slice("name", "width", "start").merge("skip" => true)
|
|
195
|
+
end
|
|
196
|
+
# 7.0 keyed the records of a layout narrowed to its columns by Strings, whatever the keys of the layout.
|
|
197
|
+
schema.delete("keyType")
|
|
198
|
+
end
|
|
199
|
+
[schema, nil]
|
|
200
|
+
rescue IOStreams::Errors::InvalidLayout, ArgumentError => e
|
|
201
|
+
[nil, "#{e.message} Write its schema by hand, see \"Moving a layout to a schema\" in IOStreams' upgrade guide"]
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Returns [IOStreams::Tabular] that reads or writes the records of this category by its schema, see #schema,
|
|
205
|
+
# reading the file that holds it when it is a path or url.
|
|
206
|
+
def schema_tabular
|
|
207
|
+
IOStreams::Tabular.new(
|
|
208
|
+
schema: IOStreams::Schema::Source.new(schema).schema,
|
|
209
|
+
format: format == :auto ? nil : format,
|
|
210
|
+
file_name: file_name
|
|
211
|
+
)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Returns [Array<Symbol>] the fields of this category that are set, which its schema replaces, see #schema.
|
|
215
|
+
def fields_replaced_by_schema
|
|
216
|
+
format_options.present? ? [:format_options] : []
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# The fields that the schema replaces cannot be set with it, since IOStreams raises for them with a schema, rather
|
|
220
|
+
# than apply one and ignore the other.
|
|
221
|
+
def schema_replaces_fields
|
|
222
|
+
return if schema.blank?
|
|
223
|
+
|
|
224
|
+
errors.add(:schema, "needs a format, such as :csv, or :auto to take it from the file name") if format.blank?
|
|
225
|
+
replaced = fields_replaced_by_schema
|
|
226
|
+
errors.add(:schema, "replaces #{replaced.to_sentence}, which cannot be set with it") unless replaced.empty?
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Checks the schema without reading a file that holds it, since its store may not be available yet.
|
|
230
|
+
def schema_valid
|
|
231
|
+
case schema
|
|
232
|
+
when nil
|
|
233
|
+
nil
|
|
234
|
+
when Hash
|
|
235
|
+
IOStreams::Schema.new(schema)
|
|
236
|
+
when String
|
|
237
|
+
schema_path_valid
|
|
238
|
+
else
|
|
239
|
+
errors.add(:schema, "must be a Hash, or the path or url of a JSON file")
|
|
240
|
+
end
|
|
241
|
+
rescue ArgumentError => e
|
|
242
|
+
errors.add(:schema, "is not valid: #{e.message}")
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def schema_path_valid
|
|
246
|
+
IOStreams.path(schema)
|
|
247
|
+
rescue LoadError
|
|
248
|
+
# A gem that the path needs is not installed in this process, so the path is checked when the job runs.
|
|
249
|
+
nil
|
|
250
|
+
rescue StandardError
|
|
251
|
+
# The path is not included in the message, since a url can include credentials.
|
|
252
|
+
errors.add(:schema, "is not a valid path or url")
|
|
253
|
+
end
|
|
79
254
|
end
|
|
80
255
|
end
|
|
81
256
|
end
|