rocketjob 7.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,6 +15,10 @@ module RocketJob
15
15
  # The header columns when the file does not include a header row.
16
16
  # Note:
17
17
  # - All column names must be strings so that it can be serialized into MongoDB.
18
+ # - With a schema, see #schema, the columns of an input category hold the header row read from the uploaded
19
+ # file, see RocketJob::Category::Input, and those of an output category cannot be set.
20
+ # - Deprecated without a schema, use `schema`, whose fields are the columns, with `"header": false` for a file
21
+ # without a header row.
18
22
  field :columns, type: Array
19
23
 
20
24
  # On an input collection `format` specifies the format of the input data so that it can be
@@ -28,17 +32,102 @@ module RocketJob
28
32
  #
29
33
  # Default: `nil`
30
34
  field :format, type: ::Mongoid::StringifiedSymbol
31
- validates_inclusion_of :format, in: [nil, :auto] + IOStreams::Tabular.registered_formats
35
+ validates_inclusion_of :format, in: [nil, :auto] + IOStreams.tabular_formats
32
36
 
33
37
  # Any specialized format specific options. For example, `:fixed` format requires a `:layout`.
38
+ #
39
+ # Deprecated, use `schema`, see #schema, which gives the width of each field of a fixed width file. It cannot be
40
+ # set with a schema.
34
41
  field :format_options, type: Hash
35
42
 
43
+ # The fields of the records in this category and the type of each one, as a Frictionless Table Schema, see
44
+ # https://datapackage.org/standard/table-schema/, so that records are read as typed values, such as an Integer,
45
+ # a BigDecimal or a Date, and written in the text form of each type, for every tabular format. It also gives
46
+ # the position of each field of a fixed width file, with its `width`, and `"header": false` describes a file
47
+ # without a header row. See `IOStreams::Schema`, and `IOStreams::Stream#schema`.
48
+ #
49
+ # A Hash, or the path or url of a JSON file that holds the schema, which is read when the category is first
50
+ # used to read or write records.
51
+ #
52
+ # Example, for a fixed width file:
53
+ # {"fields" => [{"name" => "name", "type" => "string", "width" => 10},
54
+ # {"name" => "zip", "type" => "integer", "width" => 5}]}
55
+ #
56
+ # The schema describes the columns of the file, so `format_options` and `columns` cannot be set with it, nor on
57
+ # an input category `allowed_columns`, `required_columns`, `skip_unknown` and `header_cleanser`. It needs a
58
+ # `format`, such as `:csv`, or `:auto` to take it from the file name.
59
+ #
60
+ # Default: nil, which reads and writes each value as text, as before.
61
+ field :schema
62
+ validate :schema_valid
63
+ validate :schema_replaces_fields
64
+
65
+ # Setting the schema builds the tabular again from it when it is next used, see #tabular.
66
+ define_method(:schema=) do |value|
67
+ @tabular = nil
68
+ super(value)
69
+ end
70
+
36
71
  # When `:format` is not supplied the file name can be used to infer the required format.
37
72
  # Optional.
38
73
  # Default: nil
39
74
  field :file_name, type: IOStreams::Path
40
75
  end
41
76
 
77
+ module ClassMethods
78
+ # Returns [Array<String>] the names of the deprecated options in the supplied properties of a category, with
79
+ # String keys, that its schema replaces, see #schema: its `columns`. Overridden by the input category.
80
+ def deprecated_column_options(properties)
81
+ properties["columns"].present? ? ["columns"] : []
82
+ end
83
+
84
+ # Returns [Array<String>] the names of the deprecated options of a category, see .deprecated_options, to look for
85
+ # where they cannot be read as properties, such as in the code of a RocketJob::Jobs::OnDemandBatchJob. Without
86
+ # `columns`, which a schema still uses. Overridden by the input category.
87
+ def deprecated_option_names
88
+ %w[format_options]
89
+ end
90
+
91
+ # Returns [Array<String>] the names of all the deprecated options in the supplied properties of a category,
92
+ # with String keys: its deprecated column options, see .deprecated_column_options, and `format_options`.
93
+ # Overridden by the input category.
94
+ def deprecated_options(properties)
95
+ names = deprecated_column_options(properties)
96
+ names << "format_options" if properties["format_options"].present?
97
+ names
98
+ end
99
+
100
+ # Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
101
+ # properties of a category, such as those that a job class defines it with, or that a Dirmon entry holds.
102
+ def warn_deprecated_options(properties)
103
+ properties = properties.to_h.stringify_keys
104
+ names = deprecated_column_options(properties)
105
+ unless names.empty?
106
+ options = names.collect { |name| "#{name}:" }.join(", ")
107
+ RocketJob::Deprecation.warn(
108
+ "#{options} #{names.size == 1 ? 'is' : 'are'} deprecated, describe the columns of the file with schema: " \
109
+ "instead, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
110
+ )
111
+ end
112
+ return if properties["format_options"].blank?
113
+
114
+ RocketJob::Deprecation.warn(
115
+ "format_options: is deprecated, use schema: instead, which gives the width of each field of a fixed width " \
116
+ "file, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
117
+ )
118
+ end
119
+
120
+ # Returns [Hash] the supplied attributes or properties of a category to show to people, with its file name, and
121
+ # its schema when that is the path or url of a JSON file, shown by their display names, without any
122
+ # credentials, see RocketJob.path_display_name.
123
+ def display_properties(properties)
124
+ properties.to_h do |key, value|
125
+ path = key.to_s == "file_name" || (key.to_s == "schema" && value.is_a?(String))
126
+ [key, path ? RocketJob.path_display_name(value) : value]
127
+ end
128
+ end
129
+ end
130
+
42
131
  # Return which slice serializer class to use that matches the current options.
43
132
  def serializer_class
44
133
  case serializer
@@ -58,6 +147,8 @@ module RocketJob
58
147
  end
59
148
 
60
149
  def tabular
150
+ return @tabular ||= schema_tabular if schema.present?
151
+
61
152
  @tabular ||= IOStreams::Tabular.new(
62
153
  columns: columns,
63
154
  format: format == :auto ? nil : format,
@@ -76,6 +167,90 @@ module RocketJob
76
167
  collection_name << ".#{name}" unless name == :main
77
168
  collection_name
78
169
  end
170
+
171
+ # Returns [Array<String>] the names of the deprecated options that this category sets, see .deprecated_options,
172
+ # which #schema_proposal replaces.
173
+ def deprecated_options_set
174
+ self.class.deprecated_options(attributes.to_h.stringify_keys)
175
+ end
176
+
177
+ private
178
+
179
+ # Returns [Hash] the supplied deprecated options, by name, set to their defaults, as a schema proposal sets them,
180
+ # see RocketJob::SchemaProposal#changes.
181
+ def default_settings(names)
182
+ names.to_h { |name| [name, self.class.fields[name].default_val] }
183
+ end
184
+
185
+ # Returns [Array] the schema of this category's fixed width layout, see #format_options, with String keys, as
186
+ # MongoDB stores it, and nil, or nil and why IOStreams cannot translate the layout, such as two columns with one
187
+ # key. The fields that the supplied columns leave out are skipped, which holds their place in each line.
188
+ def layout_schema(keep = nil)
189
+ layout = format_options.to_h.deep_symbolize_keys
190
+ schema = JSON.parse(JSON.generate(IOStreams::Schema.from_layout(**layout)))
191
+ if keep.present?
192
+ keep = keep.map(&:to_s)
193
+ schema["fields"] = schema["fields"].map do |field|
194
+ keep.include?(field["name"].to_s) ? field : field.slice("name", "width", "start").merge("skip" => true)
195
+ end
196
+ # 7.0 keyed the records of a layout narrowed to its columns by Strings, whatever the keys of the layout.
197
+ schema.delete("keyType")
198
+ end
199
+ [schema, nil]
200
+ rescue IOStreams::Errors::InvalidLayout, ArgumentError => e
201
+ [nil, "#{e.message} Write its schema by hand, see \"Moving a layout to a schema\" in IOStreams' upgrade guide"]
202
+ end
203
+
204
+ # Returns [IOStreams::Tabular] that reads or writes the records of this category by its schema, see #schema,
205
+ # reading the file that holds it when it is a path or url.
206
+ def schema_tabular
207
+ IOStreams::Tabular.new(
208
+ schema: IOStreams::Schema::Source.new(schema).schema,
209
+ format: format == :auto ? nil : format,
210
+ file_name: file_name
211
+ )
212
+ end
213
+
214
+ # Returns [Array<Symbol>] the fields of this category that are set, which its schema replaces, see #schema.
215
+ def fields_replaced_by_schema
216
+ format_options.present? ? [:format_options] : []
217
+ end
218
+
219
+ # The fields that the schema replaces cannot be set with it, since IOStreams raises for them with a schema, rather
220
+ # than apply one and ignore the other.
221
+ def schema_replaces_fields
222
+ return if schema.blank?
223
+
224
+ errors.add(:schema, "needs a format, such as :csv, or :auto to take it from the file name") if format.blank?
225
+ replaced = fields_replaced_by_schema
226
+ errors.add(:schema, "replaces #{replaced.to_sentence}, which cannot be set with it") unless replaced.empty?
227
+ end
228
+
229
+ # Checks the schema without reading a file that holds it, since its store may not be available yet.
230
+ def schema_valid
231
+ case schema
232
+ when nil
233
+ nil
234
+ when Hash
235
+ IOStreams::Schema.new(schema)
236
+ when String
237
+ schema_path_valid
238
+ else
239
+ errors.add(:schema, "must be a Hash, or the path or url of a JSON file")
240
+ end
241
+ rescue ArgumentError => e
242
+ errors.add(:schema, "is not valid: #{e.message}")
243
+ end
244
+
245
+ def schema_path_valid
246
+ IOStreams.path(schema)
247
+ rescue LoadError
248
+ # A gem that the path needs is not installed in this process, so the path is checked when the job runs.
249
+ nil
250
+ rescue StandardError
251
+ # The path is not included in the message, since a url can include credentials.
252
+ errors.add(:schema, "is not a valid path or url")
253
+ end
79
254
  end
80
255
  end
81
256
  end