rocketjob 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/rocket_job/batch/categories.rb +161 -40
- data/lib/rocket_job/batch/io.rb +239 -133
- data/lib/rocket_job/category/base.rb +176 -1
- data/lib/rocket_job/category/input.rb +357 -41
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +39 -0
- data/lib/rocket_job/deprecation.rb +61 -0
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +114 -20
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/plugins/retry.rb +35 -34
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +13 -6
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +5 -0
- metadata +9 -4
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
module RocketJob
|
|
2
|
+
module Category
|
|
3
|
+
# How an input category proposes the schema that replaces its deprecated options, see #schema_proposal and
|
|
4
|
+
# RocketJob::SchemaProposal, from what those options did in Rocket Job 7.0.
|
|
5
|
+
module InputSchemaProposal
|
|
6
|
+
# What to check before moving column restrictions to a schema, see #schema_proposal.
|
|
7
|
+
NON_ASCII_NOTE = "A name that 7.0 cleansed from a column with letters that are not ASCII lost them, such as " \
|
|
8
|
+
"prnom for Prénom, which the schema cleanses to prénom. Rename such a field, or the schema " \
|
|
9
|
+
"skips its column without an error. Read a recent file with #dry_run to find them.".freeze
|
|
10
|
+
|
|
11
|
+
# What to check before moving a fixed width layout to a schema, see #schema_proposal.
|
|
12
|
+
LAYOUT_NOTE = "A value that is not of the type of its field, such as letters in an integer field, raises " \
|
|
13
|
+
"IOStreams::Errors::InvalidValue, where the layout read it as 0. Each width counts the characters " \
|
|
14
|
+
"of the UTF-8 text of a line.".freeze
|
|
15
|
+
private_constant :NON_ASCII_NOTE, :LAYOUT_NOTE
|
|
16
|
+
|
|
17
|
+
# Returns [RocketJob::SchemaProposal] the schema proposed to replace this category's deprecated options at the
|
|
18
|
+
# supplied place, such as "ImportJob, input category main", or nil when it sets none. Changes nothing.
|
|
19
|
+
#
|
|
20
|
+
# The schema reads the files as 7.0 read them with those options, which is not always what they say: 7.0 did not
|
|
21
|
+
# apply `allowed_columns`, `required_columns` or `skip_unknown` to JSON records, to `columns` supplied for a file
|
|
22
|
+
# without a header row, or with `header_cleanser` :none, so they are removed there rather than described in the
|
|
23
|
+
# schema, and the proposal says so.
|
|
24
|
+
#
|
|
25
|
+
# `defined` is the category of the same name that the job class defines, for a saved job: when it has no columns,
|
|
26
|
+
# the columns of this category, in a format with a header row, hold the header row of a file, which an upload read,
|
|
27
|
+
# rather than columns that the application set. They are kept for a job that has `uploaded` its input, unless the
|
|
28
|
+
# proposed schema cannot read them, and removed otherwise, since a scheduled job copies them from its last run, so
|
|
29
|
+
# that each upload reads its own header row.
|
|
30
|
+
def schema_proposal(where, defined: nil, uploaded: false)
|
|
31
|
+
names = deprecated_options_set
|
|
32
|
+
return if names.empty?
|
|
33
|
+
|
|
34
|
+
notes = []
|
|
35
|
+
columns = self.columns
|
|
36
|
+
kept = false
|
|
37
|
+
if names.include?("columns") && defined && defined.columns.blank? && header_row_format?
|
|
38
|
+
columns = nil
|
|
39
|
+
names.delete("columns")
|
|
40
|
+
if uploaded
|
|
41
|
+
return if names.empty?
|
|
42
|
+
|
|
43
|
+
kept = true
|
|
44
|
+
else
|
|
45
|
+
names << "columns"
|
|
46
|
+
notes << copied_columns_note
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
return schema_kept_proposal(where, names) if schema.present?
|
|
50
|
+
|
|
51
|
+
schema, error = proposed_schema(columns, notes)
|
|
52
|
+
# A layout that cannot be translated keeps its settings until its schema is written by hand.
|
|
53
|
+
changes = error ? {} : default_settings(names)
|
|
54
|
+
changes["schema"] = schema if schema
|
|
55
|
+
proposal = RocketJob::SchemaProposal.new(where: where, category: self, changes: changes, notes: notes, error: error)
|
|
56
|
+
kept ? uploaded_header_proposal(proposal) : proposal
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
# Returns [String] what a proposal says of columns that it removes, which are read as the header row of an earlier
|
|
62
|
+
# upload, and the schema to use instead when the application set them for a file without a header row, unless
|
|
63
|
+
# IOStreams cannot hold such a schema, such as for columns that repeat a name.
|
|
64
|
+
def copied_columns_note
|
|
65
|
+
note = "If its columns hold the header row of an earlier upload, which a scheduled job copies to its next " \
|
|
66
|
+
"run, they are removed, so that each upload reads the header row of its own file."
|
|
67
|
+
headerless = headerless_schema(columns)
|
|
68
|
+
# Raises ArgumentError for a schema that IOStreams cannot hold.
|
|
69
|
+
IOStreams::Schema.new(headerless)
|
|
70
|
+
"#{note} If the application set them for a file without a header row, use this schema instead, which reads " \
|
|
71
|
+
"such a file as they did: #{headerless.inspect}"
|
|
72
|
+
rescue ArgumentError
|
|
73
|
+
note
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Returns [RocketJob::SchemaProposal] the supplied proposal for a job that has uploaded its input, whose workers
|
|
77
|
+
# read each line by the header row that the upload kept, see #columns, or one that changes nothing when the
|
|
78
|
+
# proposed schema cannot read that header row, such as one with two columns that are the same field, for which
|
|
79
|
+
# IOStreams raises as each slice is read.
|
|
80
|
+
def uploaded_header_proposal(proposal)
|
|
81
|
+
# The tabular of a category with a schema reads the header row that the upload kept.
|
|
82
|
+
proposal.proposed_category.tabular
|
|
83
|
+
proposal
|
|
84
|
+
rescue IOStreams::Errors::InvalidHeader => e
|
|
85
|
+
RocketJob::SchemaProposal.new(
|
|
86
|
+
where: proposal.where,
|
|
87
|
+
category: self,
|
|
88
|
+
notes: [
|
|
89
|
+
"It has uploaded its input from a header row that the proposed schema cannot read, so that each slice " \
|
|
90
|
+
"would raise #{e.class}: #{e.message}. Let it finish, or abort it, before applying a schema, rather than " \
|
|
91
|
+
"change it."
|
|
92
|
+
]
|
|
93
|
+
)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Returns [Hash] the supplied deprecated options, by name, set to their defaults, see Category::Base, and
|
|
97
|
+
# `validate_rows` set to true for the :hash mode with a schema, which parsed every line when a file was uploaded.
|
|
98
|
+
# Without a schema the :hash mode had no effect, so it is only reset.
|
|
99
|
+
def default_settings(names)
|
|
100
|
+
settings = super
|
|
101
|
+
settings["validate_rows"] = true if names.include?("mode") && mode == :hash && schema.present? && validate_rows.nil?
|
|
102
|
+
settings
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The proposal for a category that already has a schema, such as one that a Dirmon entry's properties merge into a
|
|
106
|
+
# job class with a schema, whose deprecated options are only removed.
|
|
107
|
+
def schema_kept_proposal(where, names)
|
|
108
|
+
RocketJob::SchemaProposal.new(
|
|
109
|
+
where: where,
|
|
110
|
+
category: self,
|
|
111
|
+
changes: default_settings(names),
|
|
112
|
+
notes: ["It already has a schema, so its deprecated options are removed, rather than described in another one."]
|
|
113
|
+
)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Returns [Array] the schema that replaces this category's column options and layout, read with the supplied
|
|
117
|
+
# columns, and nil, or nil and why the layout cannot be translated. Adds what to check to the supplied notes.
|
|
118
|
+
def proposed_schema(columns, notes)
|
|
119
|
+
restrictions = %w[allowed_columns required_columns].select { |name| public_send(name).present? }
|
|
120
|
+
restrictions << "skip_unknown" if skip_unknown
|
|
121
|
+
unless tabular?
|
|
122
|
+
notes << "Without a format the category's records are not parsed, so its deprecated options had no " \
|
|
123
|
+
"effect, and are removed."
|
|
124
|
+
return [nil, nil]
|
|
125
|
+
end
|
|
126
|
+
return proposed_layout_schema(columns, restrictions, notes) if format_options.present?
|
|
127
|
+
|
|
128
|
+
return keyed_schema(columns, restrictions, notes) if %i[json hash].include?(format)
|
|
129
|
+
|
|
130
|
+
if columns.present?
|
|
131
|
+
notes << never_applied(restrictions, "to the columns of a file without a header row") unless restrictions.empty?
|
|
132
|
+
if format == :auto
|
|
133
|
+
notes << "With format :auto, 7.0 read these columns as those of a CSV or PSV file without a header row, " \
|
|
134
|
+
"and narrowed the keys of a JSON file to them, while the schema describes only the former."
|
|
135
|
+
end
|
|
136
|
+
return [headerless_schema(columns), nil]
|
|
137
|
+
end
|
|
138
|
+
if header_cleanser == :none
|
|
139
|
+
notes << never_applied(restrictions, "with header_cleanser :none") unless restrictions.empty?
|
|
140
|
+
return [{"fieldsMatch" => "subset", "extraColumns" => "keep", "fields" => []}, nil]
|
|
141
|
+
end
|
|
142
|
+
if allowed_columns.blank? && required_columns.blank?
|
|
143
|
+
notes << "skip_unknown had no effect without allowed_columns, so it is removed." if skip_unknown
|
|
144
|
+
return [nil, nil]
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
notes << auto_format_note(restrictions) if format == :auto
|
|
148
|
+
notes << NON_ASCII_NOTE
|
|
149
|
+
[restricted_schema, nil]
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def proposed_layout_schema(columns, restrictions, notes)
|
|
153
|
+
unless restrictions.empty?
|
|
154
|
+
notes << "#{restrictions.join(', ')} #{restrictions.size == 1 ? 'is' : 'are'} removed with the layout. If " \
|
|
155
|
+
"they changed the records of its files, describe that in the schema, and compare with #dry_run."
|
|
156
|
+
end
|
|
157
|
+
notes << LAYOUT_NOTE
|
|
158
|
+
layout_schema(columns)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The schema of JSON records, or Hashes, whose keys 7.0 narrowed to the supplied columns, and to which it applied no
|
|
162
|
+
# column restrictions.
|
|
163
|
+
def keyed_schema(columns, restrictions, notes)
|
|
164
|
+
unless restrictions.empty?
|
|
165
|
+
notes << never_applied(restrictions, "to #{format == :json ? 'JSON records' : 'Hash records'}")
|
|
166
|
+
end
|
|
167
|
+
return [nil, nil] if columns.blank?
|
|
168
|
+
|
|
169
|
+
notes << "7.0 narrowed each record to its columns, matching its keys exactly, and gave nil for a column that a " \
|
|
170
|
+
"record did not have. The schema matches the keys whatever their case, spaces and punctuation, and " \
|
|
171
|
+
"leaves out a field that a record does not have, so read a recent file with #dry_run."
|
|
172
|
+
[{"fieldsMatch" => "partial", "fields" => columns.compact.map { |name| {"name" => name.to_s} }}, nil]
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# The schema of a file without a header row, whose columns are given, as 7.0 named them, uncleansed. A nil column
|
|
176
|
+
# is a skipped field, which holds its place.
|
|
177
|
+
def headerless_schema(columns)
|
|
178
|
+
fields = columns.each_with_index.map do |name, index|
|
|
179
|
+
name.nil? ? {"name" => "column_#{index + 1}", "skip" => true} : {"name" => name.to_s}
|
|
180
|
+
end
|
|
181
|
+
{"header" => false, "fields" => fields}
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# The schema of a file with a header row whose cleansed columns are restricted, keyed as 7.0 keyed them.
|
|
185
|
+
def restricted_schema
|
|
186
|
+
allowed = allowed_columns.to_a.map(&:to_s)
|
|
187
|
+
required = required_columns.to_a.map(&:to_s)
|
|
188
|
+
fields = (allowed | required).map do |name|
|
|
189
|
+
field = {"name" => name}
|
|
190
|
+
field.merge!("constraints" => {"required" => true}, "allowBlank" => true) if required.include?(name)
|
|
191
|
+
field
|
|
192
|
+
end
|
|
193
|
+
schema =
|
|
194
|
+
if allowed.empty?
|
|
195
|
+
# required_columns alone kept every other column of the file.
|
|
196
|
+
{"fieldsMatch" => "subset", "extraColumns" => "keep"}
|
|
197
|
+
else
|
|
198
|
+
{"fieldsMatch" => skip_unknown ? "partial" : "superset"}
|
|
199
|
+
end
|
|
200
|
+
schema.merge("fields" => fields, "keyType" => "cleansed")
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def auto_format_note(restrictions)
|
|
204
|
+
"With format :auto, 7.0 applied #{restrictions.join(', ')} to CSV and PSV files, but not to JSON files, which " \
|
|
205
|
+
"the schema also restricts, and whose keys it cleanses."
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def never_applied(names, where)
|
|
209
|
+
"7.0 did not apply #{names.join(', ')} #{where}, so #{names.size == 1 ? 'it is' : 'they are'} removed rather " \
|
|
210
|
+
"than described in the schema."
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Returns [true|false] whether this category's format reads a header row from its files, such as CSV.
|
|
214
|
+
def header_row_format?
|
|
215
|
+
IOStreams::Tabular.new(format: format == :auto ? nil : format, file_name: file_name).header?
|
|
216
|
+
rescue StandardError
|
|
217
|
+
false
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module RocketJob
|
|
2
|
+
module Category
|
|
3
|
+
# How an input category reads the lines of a text file that it uploads, see Input#each_record.
|
|
4
|
+
module InputTextLines
|
|
5
|
+
private
|
|
6
|
+
|
|
7
|
+
# Yields each line read from the supplied path, see Input#each_record. In a format the path reads the text, see
|
|
8
|
+
# Input#upload_path. Without one, IOStreams reads a line as binary, since no encoding is set on the path, which
|
|
9
|
+
# MongoDB cannot store when it holds a letter that is not ASCII, such as `José`: so the line is taken as UTF-8, without
|
|
10
|
+
# the byte order mark that a UTF-8 file can start with, which is not part of its text. A line read in an encoding
|
|
11
|
+
# set on the path, such as ISO-8859-1, is yielded as it is, and stored as UTF-8.
|
|
12
|
+
#
|
|
13
|
+
# Text that is not valid UTF-8 raises IOStreams::Errors::InvalidEncoding, naming its line, and uploads nothing,
|
|
14
|
+
# as with a schema.
|
|
15
|
+
def each_line(path, **args, &)
|
|
16
|
+
return path.each(:line, **args, &) if tabular?
|
|
17
|
+
|
|
18
|
+
path.reader(:line, **args) do |lines|
|
|
19
|
+
line_number = 1
|
|
20
|
+
lines.each do |line|
|
|
21
|
+
if line.encoding == Encoding::BINARY
|
|
22
|
+
line.force_encoding(Encoding::UTF_8)
|
|
23
|
+
line.delete_prefix!("") if line_number == 1
|
|
24
|
+
end
|
|
25
|
+
if line.encoding == Encoding::UTF_8 && !line.valid_encoding?
|
|
26
|
+
raise(IOStreams::Utils.invalid_utf8(line, line_number: line_number))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
yield(line)
|
|
30
|
+
# A line can hold a line break within quotes, such as of a CSV file, so the next one starts after it.
|
|
31
|
+
line_number = lines.line_number + 1
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -29,6 +29,45 @@ module RocketJob
|
|
|
29
29
|
slice_class: serializer_class
|
|
30
30
|
)
|
|
31
31
|
end
|
|
32
|
+
|
|
33
|
+
# Returns [RocketJob::SchemaProposal] the schema proposed to replace this category's deprecated options, its
|
|
34
|
+
# `columns` or `format_options`, at the supplied place, such as "ImportJob, output category main", or nil when it
|
|
35
|
+
# sets neither. The other keys of each record are skipped, as the columns skipped them. Changes nothing.
|
|
36
|
+
def schema_proposal(where)
|
|
37
|
+
names = deprecated_options_set
|
|
38
|
+
return if names.empty?
|
|
39
|
+
|
|
40
|
+
notes = [
|
|
41
|
+
"A Time is written in ISO 8601, such as 2026-10-13T09:30:00.123456Z. Before, it was written without its " \
|
|
42
|
+
"fraction, such as 2026-10-13 09:30:00 UTC, which a datetime field with \"format\" => " \
|
|
43
|
+
"\"%Y-%m-%d %H:%M:%S UTC\" writes.",
|
|
44
|
+
"A key of a result that only loosely matches a field, by its case or as a Symbol, is written. Before, its " \
|
|
45
|
+
"column was left blank."
|
|
46
|
+
]
|
|
47
|
+
if !tabular?
|
|
48
|
+
notes = ["Without a format the category's results are not rendered, so its deprecated options had no effect, " \
|
|
49
|
+
"and are removed."]
|
|
50
|
+
elsif format_options.present?
|
|
51
|
+
schema, error = layout_schema
|
|
52
|
+
if columns.present?
|
|
53
|
+
notes << "Its columns named the values of an Array result, which a schema writes by the position of each " \
|
|
54
|
+
"field."
|
|
55
|
+
end
|
|
56
|
+
else
|
|
57
|
+
schema = {"fieldsMatch" => "subset", "fields" => columns.map { |name| {"name" => name.to_s} }}
|
|
58
|
+
end
|
|
59
|
+
# A layout that cannot be translated keeps its settings until its schema is written by hand.
|
|
60
|
+
changes = error ? {} : default_settings(names)
|
|
61
|
+
changes["schema"] = schema if schema
|
|
62
|
+
RocketJob::SchemaProposal.new(where: where, category: self, changes: changes, notes: notes, error: error)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# The schema gives the columns of the file written, see Category::Base#schema.
|
|
68
|
+
def fields_replaced_by_schema
|
|
69
|
+
columns.present? ? super + [:columns] : super
|
|
70
|
+
end
|
|
32
71
|
end
|
|
33
72
|
end
|
|
34
73
|
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
module RocketJob
|
|
2
|
+
# Reports the use of a deprecated option or method as a Ruby deprecation warning, see `Kernel#warn`, as IOStreams
|
|
3
|
+
# does, see `IOStreams::Deprecation`:
|
|
4
|
+
# - Only while deprecation warnings are on, see `Warning[:deprecated]`. They are off by default, so production stays
|
|
5
|
+
# quiet, and on in tests run by Minitest or test-unit, or with `ruby -W:deprecated` or `RUBYOPT=-W:deprecated`.
|
|
6
|
+
# - Written to `$stderr`, unless the application routes them elsewhere, such as to its log, or raises them in its
|
|
7
|
+
# tests, by overriding `Warning.warn`.
|
|
8
|
+
#
|
|
9
|
+
# Each warning names the first line of the call stack outside Rocket Job, such as the line of a job class that
|
|
10
|
+
# defines a category with a deprecated option, rather than a line of Rocket Job.
|
|
11
|
+
module Deprecation
|
|
12
|
+
# The directory of the files of this library, whose lines are skipped when finding the line that used it.
|
|
13
|
+
LIB_DIR = "#{File.expand_path('..', __dir__)}/".freeze
|
|
14
|
+
|
|
15
|
+
# Where the upgrade guide says what replaces each option that 7.1 deprecates.
|
|
16
|
+
UPGRADE_GUIDE = "https://rocketjob.reidmorrison.com/upgrading.html#upgrading-to-v71".freeze
|
|
17
|
+
|
|
18
|
+
# Where the upgrade guide says what replaces each option that 7.2 deprecates.
|
|
19
|
+
UPGRADE_GUIDE_V72 = "https://rocketjob.reidmorrison.com/upgrading.html#upgrading-to-v72".freeze
|
|
20
|
+
|
|
21
|
+
# Reports that the application used something deprecated, unless deprecation warnings are off, with the message,
|
|
22
|
+
# which says what is deprecated and what to use instead, after "Rocket Job: ", at the first line of the call stack
|
|
23
|
+
# outside this library and the libraries that call it, or without a line when there is none, such as in a worker
|
|
24
|
+
# that builds the job of a Dirmon entry.
|
|
25
|
+
#
|
|
26
|
+
# Returns nil.
|
|
27
|
+
def self.warn(message)
|
|
28
|
+
return unless enabled?
|
|
29
|
+
|
|
30
|
+
dirs = library_dirs
|
|
31
|
+
uplevel = caller_locations(1).index { |location| !library?(location, dirs) }
|
|
32
|
+
# Kernel#warn counts its own caller, this method, as level 0.
|
|
33
|
+
Kernel.warn("Rocket Job: #{message}", uplevel: uplevel && (uplevel + 1), category: :deprecated)
|
|
34
|
+
nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Returns [true|false] whether deprecation warnings are on, see Warning[:deprecated], and so whether .warn reports
|
|
38
|
+
# them. A check that costs something to find a use of what is deprecated, such as one that compares every record
|
|
39
|
+
# of an upload, runs only while they are, so that production, where they are off, does not pay for it.
|
|
40
|
+
def self.enabled?
|
|
41
|
+
Warning[:deprecated]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Returns [Array<String>] the directories whose lines are skipped when finding the line of the application: this
|
|
45
|
+
# library, the other gems, which call it back, such as Active Support's callbacks and Mongoid, and Ruby's own
|
|
46
|
+
# library.
|
|
47
|
+
def self.library_dirs
|
|
48
|
+
gems = Gem.loaded_specs.each_value.filter_map { |spec| "#{spec.full_gem_path}/" unless spec.name == "rocketjob" }
|
|
49
|
+
[LIB_DIR, "#{RbConfig::CONFIG['rubylibdir']}/", *gems]
|
|
50
|
+
end
|
|
51
|
+
private_class_method :library_dirs
|
|
52
|
+
|
|
53
|
+
# Returns [true|false] whether the line of the call stack is in a file of a library, see .library_dirs, or of Ruby
|
|
54
|
+
# itself.
|
|
55
|
+
def self.library?(location, dirs)
|
|
56
|
+
path = (location.absolute_path || location.path).to_s
|
|
57
|
+
path.start_with?("<internal:") || dirs.any? { |dir| path.start_with?(dir) }
|
|
58
|
+
end
|
|
59
|
+
private_class_method :library?
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -186,13 +186,13 @@ module RocketJob
|
|
|
186
186
|
|
|
187
187
|
# Security check?
|
|
188
188
|
if whitelist_paths.size.positive? && whitelist_paths.none? { |whitepath| path.to_s.start_with?(whitepath) }
|
|
189
|
-
logger.warn "Skipping file: #{path} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
|
|
189
|
+
logger.warn "Skipping file: #{file_display_name(path)} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
|
|
190
190
|
next
|
|
191
191
|
end
|
|
192
192
|
|
|
193
193
|
# File must be writable so it can be removed after processing
|
|
194
194
|
if path.respond_to?(:writable?) && !path.writable?
|
|
195
|
-
logger.warn "Skipping file: #{
|
|
195
|
+
logger.warn "Skipping file: #{file_display_name(path)} since it is not writable by the current user. Must be able to delete/move the file after queueing the job"
|
|
196
196
|
next
|
|
197
197
|
end
|
|
198
198
|
yield(path)
|
|
@@ -215,6 +215,72 @@ module RocketJob
|
|
|
215
215
|
end
|
|
216
216
|
end
|
|
217
217
|
|
|
218
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated category options in the
|
|
219
|
+
# properties of this entry, which its job class proposes, see
|
|
220
|
+
# RocketJob::Batch::Categories::ClassMethods#property_schema_proposals. None when its job class is not loaded, or is not a Rocket Job.
|
|
221
|
+
# What its job class raises for the properties, such as for a category option that it no longer has, is answered as
|
|
222
|
+
# one proposal with the error, see RocketJob::SchemaProposal.unreadable.
|
|
223
|
+
def schema_proposals
|
|
224
|
+
where = "Dirmon entry #{name.inspect} #{id}"
|
|
225
|
+
klass = rocket_job_class
|
|
226
|
+
return [] unless klass && properties.is_a?(Hash)
|
|
227
|
+
|
|
228
|
+
klass.property_schema_proposals(properties, where)
|
|
229
|
+
rescue StandardError => e
|
|
230
|
+
[RocketJob::SchemaProposal.unreadable(where, e)]
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Returns [String] the pattern to show, for example in a web interface, which leaves out any credentials,
|
|
234
|
+
# such as the user name and password of `sftp://user:password@host/in/*.csv`, see IOStreams::Path#display_name.
|
|
235
|
+
#
|
|
236
|
+
# A url cannot hold some pattern characters, such as the braces of `sftp://host/in/*.{csv,txt}`. Such a pattern
|
|
237
|
+
# is shown as the display name of the directory that is scanned, followed by the pattern within it,
|
|
238
|
+
# the same way that IOStreams.each_child splits it.
|
|
239
|
+
#
|
|
240
|
+
# Unless the pattern within the directory has a `?`, which cannot be told apart from the query of a url, such as
|
|
241
|
+
# `sftp://host/in/*.{csv,txt}?password=secret`, where the query holds ssh options, which could hold a secret.
|
|
242
|
+
# Nor when the pattern within the directory would start within the user name, password or host of a url, such as at
|
|
243
|
+
# the brace in the password of `sftp://user:pa{ss@host/in/*.csv`, since it would show them.
|
|
244
|
+
def pattern_display_name
|
|
245
|
+
display_name = RocketJob.path_display_name(pattern)
|
|
246
|
+
return display_name unless display_name == RocketJob::INVALID_PATH_DISPLAY_NAME
|
|
247
|
+
|
|
248
|
+
matcher = IOStreams::Paths::Matcher.new(nil, pattern)
|
|
249
|
+
return display_name if matcher.pattern.nil? || matcher.pattern.include?("?") || !split_after_authority?(matcher)
|
|
250
|
+
|
|
251
|
+
"#{matcher.path.display_name}/#{matcher.pattern}"
|
|
252
|
+
rescue StandardError, LoadError
|
|
253
|
+
# The directory is not a valid path, or needs a gem that is not installed in this process.
|
|
254
|
+
RocketJob::INVALID_PATH_DISPLAY_NAME
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Returns [String] the archive directory to show, without any credentials, see #pattern_display_name.
|
|
258
|
+
def archive_directory_display_name
|
|
259
|
+
RocketJob.path_display_name(archive_directory)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Returns [Hash] the properties to show, with each path or url that the job class holds in them shown without
|
|
263
|
+
# any credentials, see RocketJob::Plugins::Job::Model.display_properties. The properties are shown as stored when
|
|
264
|
+
# the job class is not defined, or is not a Rocket Job, since then it cannot say which of them are paths.
|
|
265
|
+
def display_properties
|
|
266
|
+
klass = rocket_job_class
|
|
267
|
+
klass ? klass.display_properties(properties) : properties
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Returns [RocketJob::DirmonEntry] a new, unsaved, pending entry with the settings of this entry, such as its
|
|
271
|
+
# job class and properties, overridden by the supplied attributes, which usually include a new name and pattern.
|
|
272
|
+
#
|
|
273
|
+
# Only the settings in REPLICATED_ATTRIBUTES are copied, not what happened to this entry, such as its state,
|
|
274
|
+
# or why it failed.
|
|
275
|
+
#
|
|
276
|
+
# The name and pattern must each be unique, so the replica is not valid until both are changed. They are copied
|
|
277
|
+
# so that, for example, a web interface can show them to be edited.
|
|
278
|
+
def replicate(**attributes)
|
|
279
|
+
entry = self.class.new(self.attributes.slice(*REPLICATED_ATTRIBUTES).deep_dup)
|
|
280
|
+
entry.assign_attributes(attributes)
|
|
281
|
+
entry
|
|
282
|
+
end
|
|
283
|
+
|
|
218
284
|
# Returns the Job to be created.
|
|
219
285
|
def job_class
|
|
220
286
|
return if job_class_name.nil?
|
|
@@ -230,12 +296,13 @@ module RocketJob
|
|
|
230
296
|
archive_path = archive_iopath(iopath).join("#{job_id}_#{iopath.basename}")
|
|
231
297
|
iopath.move_to(archive_path)
|
|
232
298
|
|
|
233
|
-
|
|
299
|
+
original_file_name = file_display_name(iopath)
|
|
300
|
+
job = RocketJob::Jobs::UploadFileJob.create!(
|
|
234
301
|
job_class_name: job_class_name,
|
|
235
302
|
properties: properties,
|
|
236
303
|
description: "#{name}: #{iopath.basename}",
|
|
237
304
|
upload_file_name: archive_path,
|
|
238
|
-
original_file_name:
|
|
305
|
+
original_file_name: original_file_name,
|
|
239
306
|
job_id: job_id
|
|
240
307
|
)
|
|
241
308
|
|
|
@@ -243,8 +310,8 @@ module RocketJob
|
|
|
243
310
|
message: "Created RocketJob::Jobs::UploadFileJob",
|
|
244
311
|
payload: {
|
|
245
312
|
dirmon_entry_name: name,
|
|
246
|
-
upload_file_name: archive_path,
|
|
247
|
-
original_file_name:
|
|
313
|
+
upload_file_name: archive_path.display_name,
|
|
314
|
+
original_file_name: original_file_name,
|
|
248
315
|
job_class_name: job_class_name,
|
|
249
316
|
job_id: job_id.to_s,
|
|
250
317
|
upload_job_id: job.id.to_s
|
|
@@ -253,8 +320,53 @@ module RocketJob
|
|
|
253
320
|
job
|
|
254
321
|
end
|
|
255
322
|
|
|
323
|
+
# Returns [String] the name of a file that this entry found, to save, log and show, without any credentials, so
|
|
324
|
+
# that each file has a name of its own, and the upload of a file reads it by its extensions, such as `.csv.gz`, see
|
|
325
|
+
# RocketJob::Jobs::UploadFileJob#original_file_name.
|
|
326
|
+
#
|
|
327
|
+
# A local file is named by its display name, see IOStreams::Path#display_name, which shows each byte of its name
|
|
328
|
+
# that is not valid UTF-8 as `\xHH`. A file on a server, such as an SFTP or S3 file, is named by its url without
|
|
329
|
+
# any user name, password or query, with its path percent-encoded, which IOStreams reads back as the same path.
|
|
330
|
+
# Without it IOStreams 2.2 cannot read back the url of a file whose name holds a non-ASCII letter or a `[`, and
|
|
331
|
+
# reads a `+` in it as a space, a `%` followed by two hexadecimal digits as the byte they encode, and a `#` or `?`
|
|
332
|
+
# as the start of a fragment or query. For example the SFTP file `/in/données#1.csv.gz` is named
|
|
333
|
+
# `sftp://host/in/donn%C3%A9es%231.csv.gz`. The url of a file whose name holds only letters, digits and characters
|
|
334
|
+
# such as `-`, `_` and `.` is not changed.
|
|
335
|
+
#
|
|
336
|
+
# The name is valid UTF-8, so that it can be saved, see RocketJob.display_text.
|
|
337
|
+
def file_display_name(iopath)
|
|
338
|
+
name = RocketJob.display_text(iopath.display_name)
|
|
339
|
+
# The scheme, host and port of a url, which its display name holds without any user name or password.
|
|
340
|
+
location = name[%r{\A[^:/?#]+://[^/?#]*}]
|
|
341
|
+
return name unless location
|
|
342
|
+
|
|
343
|
+
"#{location}/#{url_path(iopath.path).delete_prefix('/')}"
|
|
344
|
+
end
|
|
345
|
+
|
|
256
346
|
private
|
|
257
347
|
|
|
348
|
+
# The settings of an entry, which #replicate copies. A new field is not copied unless it is added here.
|
|
349
|
+
REPLICATED_ATTRIBUTES = %w[name pattern job_class_name properties archive_directory].freeze
|
|
350
|
+
private_constant :REPLICATED_ATTRIBUTES
|
|
351
|
+
|
|
352
|
+
# Whether the matcher splits the pattern into a directory, and the pattern within it, after the scheme, user name,
|
|
353
|
+
# password and host of a url, so that the pattern within the directory holds none of them, see
|
|
354
|
+
# #pattern_display_name. A pattern that is not a url, such as a local path, has none of them.
|
|
355
|
+
def split_after_authority?(matcher)
|
|
356
|
+
directory = pattern.delete_suffix("/#{matcher.pattern}")
|
|
357
|
+
return false if directory == pattern
|
|
358
|
+
|
|
359
|
+
authority = pattern[%r{\A[^:/]+://[^/]*}]
|
|
360
|
+
authority.nil? || directory.start_with?(authority)
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Returns [String] the supplied path of a file with each byte that a url cannot hold in its path percent-encoded,
|
|
364
|
+
# such as a `#`, `?`, `%`, `+`, `[`, space, non-ASCII letter, or a byte that is not valid UTF-8. So that it is valid
|
|
365
|
+
# UTF-8, and IOStreams reads it back as the same path, see #file_display_name.
|
|
366
|
+
def url_path(path)
|
|
367
|
+
path.to_s.b.gsub(%r{[^A-Za-z0-9\-._~!$&'()*,;=:@/]}n) { |byte| format("%%%02X", byte.ord) }.force_encoding(Encoding::UTF_8)
|
|
368
|
+
end
|
|
369
|
+
|
|
258
370
|
# strip whitespaces from all variables that reference paths or patterns
|
|
259
371
|
def strip_whitespace
|
|
260
372
|
self.pattern = pattern.strip unless pattern.nil?
|
|
@@ -273,6 +385,14 @@ module RocketJob
|
|
|
273
385
|
path.relative? ? iopath.directory.join(archive_directory) : path
|
|
274
386
|
end
|
|
275
387
|
|
|
388
|
+
# Returns [Class] the job class when it is defined and is a Rocket Job, otherwise nil, such as for a job class
|
|
389
|
+
# name that a person entered, which names another class or constant, so that it is only asked what a Rocket Job
|
|
390
|
+
# answers, such as #display_properties.
|
|
391
|
+
def rocket_job_class
|
|
392
|
+
klass = job_class
|
|
393
|
+
klass if klass.is_a?(Class) && klass <= RocketJob::Job
|
|
394
|
+
end
|
|
395
|
+
|
|
276
396
|
# Validates job_class is a Rocket Job
|
|
277
397
|
def job_is_a_rocket_job
|
|
278
398
|
klass = job_class
|
|
@@ -7,10 +7,34 @@ module IOStreams
|
|
|
7
7
|
|
|
8
8
|
# Get the object as it was stored in the database, and instantiate
|
|
9
9
|
# this custom class from it.
|
|
10
|
+
#
|
|
11
|
+
# A stored url that IOStreams cannot make a path of is returned as stored, a String, rather than raising, so that
|
|
12
|
+
# the name can still be read, such as for its extensions, or to show it, see RocketJob.path_display_name. Such as:
|
|
13
|
+
# - a url that IOStreams cannot parse, such as an SFTP url with a non-ASCII letter in its file name. For example
|
|
14
|
+
# the `file_name` of a category, which an UploadFileJob that Dirmon created in Rocket Job 7.1 set from the
|
|
15
|
+
# unencoded url of an SFTP file, see RocketJob::DirmonEntry#file_display_name.
|
|
16
|
+
# - a url whose scheme needs a gem that this process does not load, such as an S3 url in a web process without
|
|
17
|
+
# aws-sdk-s3, while the workers that read the file load it.
|
|
18
|
+
# - a url whose scheme IOStreams does not know, such as one that the application registers in another process.
|
|
19
|
+
# IOStreams raises when the String is used as a path, such as with IOStreams.new.
|
|
10
20
|
def self.demongoize(object)
|
|
11
21
|
return if object.nil?
|
|
12
22
|
|
|
13
23
|
IOStreams.new(object)
|
|
24
|
+
rescue ArgumentError, URI::Error, LoadError
|
|
25
|
+
object
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Returns [true|false] whether IOStreams cannot make a path of the supplied url in this process, other than for a
|
|
29
|
+
# gem that this process does not load, since a server that runs the job can load it: such as a url whose scheme
|
|
30
|
+
# IOStreams does not know, or that it cannot parse, which .demongoize keeps as stored.
|
|
31
|
+
def self.invalid_url?(url)
|
|
32
|
+
IOStreams.path(url)
|
|
33
|
+
false
|
|
34
|
+
rescue ArgumentError, URI::Error
|
|
35
|
+
true
|
|
36
|
+
rescue LoadError
|
|
37
|
+
false
|
|
14
38
|
end
|
|
15
39
|
|
|
16
40
|
# Takes any possible object and converts it to how it would be
|