rocketjob_mission_control 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/rocket_job_mission_control/dirmon_entries_controller.rb +15 -7
- data/app/controllers/rocket_job_mission_control/jobs_controller.rb +3 -3
- data/app/datatables/rocket_job_mission_control/dirmon_entries_datatable.rb +1 -1
- data/app/helpers/rocket_job_mission_control/application_helper.rb +10 -0
- data/app/helpers/rocket_job_mission_control/dirmon_entries_helper.rb +16 -0
- data/app/helpers/rocket_job_mission_control/jobs_helper.rb +16 -3
- data/app/models/rocket_job_mission_control/dirmon_sanitizer.rb +119 -2
- data/app/models/rocket_job_mission_control/job_sanitizer.rb +50 -16
- data/app/views/rocket_job_mission_control/application/_schema_proposals.html.erb +74 -0
- data/app/views/rocket_job_mission_control/dirmon_entries/_details.html.erb +2 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/_form.html.erb +3 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/_input_category_fields.html.erb +5 -5
- data/app/views/rocket_job_mission_control/dirmon_entries/_output_category_fields.html.erb +2 -1
- data/app/views/rocket_job_mission_control/dirmon_entries/_properties.html.erb +3 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/show.html.erb +1 -0
- data/app/views/rocket_job_mission_control/jobs/_details.html.erb +1 -1
- data/app/views/rocket_job_mission_control/jobs/_input_categories.html.erb +4 -4
- data/app/views/rocket_job_mission_control/jobs/_input_category_fields.html.erb +3 -4
- data/app/views/rocket_job_mission_control/jobs/_output_categories.html.erb +1 -1
- data/app/views/rocket_job_mission_control/jobs/show.html.erb +1 -0
- data/lib/rocket_job_mission_control/version.rb +1 -1
- metadata +5 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6fb6310a1a33fad99189a35c059a32f6eb77222f27276ca490eb80d61230e1c0
|
|
4
|
+
data.tar.gz: 316fb96d05902f8399717fc0cec7f7987c86906096ad209d40b297d2aad8e90d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 427be2abefed3b1eb0845b1fd26dd6258d4dbe06cc43c1cb23f4a5c2ff43d8192b9a6eb394027fd5ea43616543f23f62b2d88f5ff0f36d6c572a0a50bff23b2f
|
|
7
|
+
data.tar.gz: 074d0349f4df4001889ff54809005d5e2fccce394ada49ce4d534147e7c2ca8fb6fdfd3c54500d194fa6672d29629d2eb582d7607996a60ebea7388613403a30
|
|
@@ -68,10 +68,11 @@ module RocketJobMissionControl
|
|
|
68
68
|
|
|
69
69
|
def update
|
|
70
70
|
authorize! :update, @dirmon_entry
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
entry_params = params[:rocket_job_dirmon_entry]
|
|
72
|
+
sanitized_params = DirmonSanitizer.sanitize(entry_params, @dirmon_entry.job_class, @dirmon_entry)
|
|
73
73
|
|
|
74
|
-
sanitized_params[:properties] =
|
|
74
|
+
sanitized_params[:properties] =
|
|
75
|
+
DirmonSanitizer.update_properties(entry_params[:properties], sanitized_params[:properties], @dirmon_entry)
|
|
75
76
|
|
|
76
77
|
if @dirmon_entry.errors.empty? && @dirmon_entry.valid? && @dirmon_entry.update(sanitized_params)
|
|
77
78
|
redirect_to(rocket_job_mission_control.dirmon_entry_path(@dirmon_entry))
|
|
@@ -96,11 +97,11 @@ module RocketJobMissionControl
|
|
|
96
97
|
# the replicate method clones the existing Dirmon Entity
|
|
97
98
|
def replicate
|
|
98
99
|
authorize! :replicate, @dirmon_entry
|
|
99
|
-
dirmon_entry_replicate =
|
|
100
|
+
dirmon_entry_replicate = @dirmon_entry.replicate
|
|
100
101
|
|
|
101
102
|
if (properties = params[:rocket_job_dirmon_entry][:properties])
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
sanitized = DirmonSanitizer.sanitize_properties(properties, dirmon_entry_replicate.job_class, @dirmon_entry)
|
|
104
|
+
dirmon_entry_replicate.properties = DirmonSanitizer.update_properties(properties, sanitized, @dirmon_entry)
|
|
104
105
|
end
|
|
105
106
|
|
|
106
107
|
if dirmon_entry_replicate.update(dirmon_params)
|
|
@@ -160,6 +161,13 @@ module RocketJobMissionControl
|
|
|
160
161
|
params.fetch(:rocket_job_dirmon_entry, {}).permit(:name, :archive_directory, :pattern, :job_class_name, :properties)
|
|
161
162
|
end
|
|
162
163
|
|
|
164
|
+
# The columns of the Dirmon entries list that its search box searches. A pattern can hold credentials, which the
|
|
165
|
+
# list leaves out, see RocketJob::DirmonEntry#pattern_display_name, so it is only searched for a role that can edit
|
|
166
|
+
# an entry, whose form shows the pattern as stored. Otherwise a search could find a password one letter at a time.
|
|
167
|
+
def dirmon_search_columns
|
|
168
|
+
can?(:edit, RocketJob::DirmonEntry) ? %i[job_class_name name pattern] : %i[job_class_name name]
|
|
169
|
+
end
|
|
170
|
+
|
|
163
171
|
def render_datatable(entries, description)
|
|
164
172
|
respond_to do |format|
|
|
165
173
|
format.html do
|
|
@@ -169,7 +177,7 @@ module RocketJobMissionControl
|
|
|
169
177
|
|
|
170
178
|
format.json do
|
|
171
179
|
query = RocketJobMissionControl::Query.new(entries, name: :asc)
|
|
172
|
-
query.search_columns =
|
|
180
|
+
query.search_columns = dirmon_search_columns
|
|
173
181
|
query.display_columns = %w[name _type pattern]
|
|
174
182
|
render(json: DirmonEntriesDatatable.new(view_context, query))
|
|
175
183
|
end
|
|
@@ -188,7 +188,7 @@ module RocketJobMissionControl
|
|
|
188
188
|
slice.records = records
|
|
189
189
|
|
|
190
190
|
if slice.save
|
|
191
|
-
logger.info("Slice Updated By #{login}, job: #{@job.id}, file_name: #{@job
|
|
191
|
+
logger.info("Slice Updated By #{login}, job: #{@job.id}, file_name: #{helpers.job_upload_file_name(@job)}")
|
|
192
192
|
flash[:success] = "slice updated"
|
|
193
193
|
redirect_to view_slice_job_path(@job, error_type: error_type, offset: offset)
|
|
194
194
|
else
|
|
@@ -220,7 +220,7 @@ module RocketJobMissionControl
|
|
|
220
220
|
slice.records = records
|
|
221
221
|
|
|
222
222
|
if slice.save
|
|
223
|
-
logger.info("Line Deleted By #{login}, job: #{@job.id}, file_name: #{@job
|
|
223
|
+
logger.info("Line Deleted By #{login}, job: #{@job.id}, file_name: #{helpers.job_upload_file_name(@job)}")
|
|
224
224
|
flash[:success] = "Record #{slice.first_record_number + line_index} removed from the slice."
|
|
225
225
|
redirect_to view_slice_job_path(@job, error_type: error_type, offset: offset), status: :see_other
|
|
226
226
|
else
|
|
@@ -300,7 +300,7 @@ module RocketJobMissionControl
|
|
|
300
300
|
:format,
|
|
301
301
|
:format_options,
|
|
302
302
|
:schema,
|
|
303
|
-
:
|
|
303
|
+
:validate_rows,
|
|
304
304
|
:skip_unknown,
|
|
305
305
|
:slice_size,
|
|
306
306
|
{columns: []}
|
|
@@ -8,7 +8,7 @@ module RocketJobMissionControl
|
|
|
8
8
|
{
|
|
9
9
|
"0" => name_with_link(dirmon),
|
|
10
10
|
"1" => h(dirmon.job_class_name),
|
|
11
|
-
"2" => h(dirmon.
|
|
11
|
+
"2" => h(dirmon.pattern_display_name.try(:truncate, 80)),
|
|
12
12
|
"DT_RowClass" => "rjmc-card callout callout-#{dirmon.state}"
|
|
13
13
|
}
|
|
14
14
|
end
|
|
@@ -91,6 +91,16 @@ module RocketJobMissionControl
|
|
|
91
91
|
placeholder: '{"fields": [{"name": "name"}, {"name": "age", "type": "integer"}]}')
|
|
92
92
|
end
|
|
93
93
|
|
|
94
|
+
# Returns the select for the validate_rows of an input category, see RocketJob::Category::Input#validate_rows:
|
|
95
|
+
# blank, which clears it on a job and leaves it to the job class on a Dirmon entry, true or false. A category without
|
|
96
|
+
# one whose upload validates its lines anyway, such as one with a schema in the deprecated :hash mode, shows true,
|
|
97
|
+
# as Rocket Job 8 reads it, so that the form shows what the category does.
|
|
98
|
+
def validate_rows_select(f, category)
|
|
99
|
+
value = category.validate_rows
|
|
100
|
+
value = true if value.nil? && category.upload_validates_rows?
|
|
101
|
+
f.select(:validate_rows, [true, false], {include_blank: true, selected: value}, {class: "tom-select form-select"})
|
|
102
|
+
end
|
|
103
|
+
|
|
94
104
|
# Returns the editable field as html for use in editing dynamic fields from a Job class.
|
|
95
105
|
def editable_field_html(klass, field_name, value, f)
|
|
96
106
|
# When editing a job the values are of the correct type.
|
|
@@ -11,6 +11,22 @@ module RocketJobMissionControl
|
|
|
11
11
|
nil
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
# Returns [RocketJob::Job] the job that the entry's properties build, whose values its form shows, or nil without a
|
|
15
|
+
# job class. Returns nil too when Rocket Job cannot build one from them, such as for a category that the job class
|
|
16
|
+
# does not define or an option that it no longer has, and adds why to the entry's errors, so that its form still
|
|
17
|
+
# shows its other attributes rather than failing. The error is logged with its backtrace too, since it can come from
|
|
18
|
+
# the job class itself, such as a mistake in a default of one of its fields.
|
|
19
|
+
def dirmon_entry_job(dirmon_entry)
|
|
20
|
+
dirmon_entry.job_class&.from_properties(dirmon_entry.properties)
|
|
21
|
+
rescue ArgumentError, NoMethodError, Mongoid::Errors::MongoidError => e
|
|
22
|
+
job_class_name = dirmon_entry.job_class_name
|
|
23
|
+
logger.warn("Dirmon entry #{dirmon_entry.id}: its properties cannot build a #{job_class_name}: #{e.class}: " \
|
|
24
|
+
"#{e.message}\n#{Array(e.backtrace).join("\n")}")
|
|
25
|
+
reason = e.respond_to?(:problem) ? e.problem : e.message
|
|
26
|
+
dirmon_entry.errors.add(:properties, "cannot build a #{job_class_name}, so they cannot be edited here: #{reason}")
|
|
27
|
+
nil
|
|
28
|
+
end
|
|
29
|
+
|
|
14
30
|
def rocket_job_mission_control
|
|
15
31
|
@@rocket_job_mission_control_engine_url_helpers ||= RocketJobMissionControl::Engine.routes.url_helpers
|
|
16
32
|
end
|
|
@@ -107,14 +107,27 @@ module RocketJobMissionControl
|
|
|
107
107
|
(job.record_count.to_f / job.input_category.slice_size).ceil
|
|
108
108
|
end
|
|
109
109
|
|
|
110
|
-
# The
|
|
110
|
+
# The job's upload file name, without any credentials, such as the password of an SFTP url that Dirmon
|
|
111
|
+
# archived the file to, see RocketJob.path_display_name.
|
|
112
|
+
def job_upload_file_name(job)
|
|
113
|
+
RocketJob.path_display_name(job.upload_file_name)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# The category's file name, without any credentials, such as the password of an SFTP url that the output is
|
|
117
|
+
# downloaded to. The category decides how its file name is shown, see RocketJob::Category::Base.display_properties.
|
|
118
|
+
def job_category_file_name(category)
|
|
119
|
+
category.class.display_properties("file_name" => category.file_name)["file_name"]
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The category's schema: its fields as a JSON tree, or the path or url of the JSON file that holds it, without any
|
|
123
|
+
# credentials. The category decides how its schema is shown, see RocketJob::Category::Base.display_properties.
|
|
111
124
|
def job_category_schema(category)
|
|
112
|
-
schema = category.schema
|
|
125
|
+
schema = category.class.display_properties("schema" => category.schema)["schema"]
|
|
113
126
|
schema.is_a?(Hash) ? render_json_tree(schema) : schema
|
|
114
127
|
end
|
|
115
128
|
|
|
116
129
|
def job_custom_fields(job)
|
|
117
|
-
attrs = job.
|
|
130
|
+
attrs = job.display_attributes
|
|
118
131
|
DISPLAYED_FIELDS.each { |key| attrs.delete(key) }
|
|
119
132
|
# Convert time zones for any custom time fields
|
|
120
133
|
attrs.keys { |key| attrs[key] = attrs[key].in_time_zone(Time.zone) if attrs[key].is_a?(Time) }
|
|
@@ -13,13 +13,47 @@ module RocketJobMissionControl
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
if params.key?(:properties)
|
|
16
|
-
properties =
|
|
16
|
+
properties = sanitize_properties(params[:properties], job_class, target)
|
|
17
17
|
permissible_params[:properties] = properties if properties.present?
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
permissible_params
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# Returns [Hash] the supplied properties that a form posted, sanitized for a Dirmon entry, see JobSanitizer.sanitize,
|
|
24
|
+
# without those posted blank, which .update_properties removes from the entry.
|
|
25
|
+
def self.sanitize_properties(posted, job_class, target)
|
|
26
|
+
JobSanitizer.sanitize(posted, job_class, target, false)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Returns [Hash] the properties of the supplied Dirmon entry with the changes that its form posted, see
|
|
30
|
+
# .diff_properties: each property that the form posted is set when it differs from the job class's default, and
|
|
31
|
+
# removed otherwise, so that the job class decides it. Every other property is kept as it is saved, such as an
|
|
32
|
+
# input category's `allowed_columns` or `file_name`, since the form does not show it.
|
|
33
|
+
#
|
|
34
|
+
# Parameters
|
|
35
|
+
# posted [Hash]
|
|
36
|
+
# The properties that the form posted, see JobSanitizer.posted_fields and .posted_categories.
|
|
37
|
+
#
|
|
38
|
+
# sanitized [Hash]
|
|
39
|
+
# The same properties after .sanitize_properties.
|
|
40
|
+
#
|
|
41
|
+
# dirmon_entry [RocketJob::DirmonEntry]
|
|
42
|
+
# The entry whose properties the form shows, such as the one being edited or copied.
|
|
43
|
+
def self.update_properties(posted, sanitized, dirmon_entry)
|
|
44
|
+
properties = dirmon_entry.properties.to_h.deep_stringify_keys
|
|
45
|
+
return properties if posted.blank?
|
|
46
|
+
|
|
47
|
+
changes = diff_properties(sanitized, dirmon_entry).deep_stringify_keys
|
|
48
|
+
JobSanitizer.posted_fields(posted, dirmon_entry.job_class).each do |name|
|
|
49
|
+
merge_property(properties, changes, name.to_s)
|
|
50
|
+
end
|
|
51
|
+
JobSanitizer.posted_categories(posted).each do |category|
|
|
52
|
+
merge_category(properties, category, changes, dirmon_entry.job_class)
|
|
53
|
+
end
|
|
54
|
+
properties
|
|
55
|
+
end
|
|
56
|
+
|
|
23
57
|
# Returns [Hash] the difference between the supplied params and those already set in the job itself
|
|
24
58
|
def self.diff_properties(sanitized_properties, dirmon_entry)
|
|
25
59
|
default_job = dirmon_entry.job_class.new
|
|
@@ -46,7 +80,7 @@ module RocketJobMissionControl
|
|
|
46
80
|
end
|
|
47
81
|
properties[:output_categories] = categories unless categories.empty?
|
|
48
82
|
elsif default_job.public_send(name) != updated_job.public_send(name)
|
|
49
|
-
properties[name] = value
|
|
83
|
+
properties[name] = value
|
|
50
84
|
end
|
|
51
85
|
end
|
|
52
86
|
properties
|
|
@@ -67,5 +101,88 @@ module RocketJobMissionControl
|
|
|
67
101
|
diff[:name] = name unless diff.empty?
|
|
68
102
|
diff
|
|
69
103
|
end
|
|
104
|
+
|
|
105
|
+
# Merges the changes that the form posted for the supplied category, see JobSanitizer.posted_categories, into the
|
|
106
|
+
# supplied properties, see .update_properties. A saved category is found by its name, or is main without one, as a
|
|
107
|
+
# job reads it, and is removed when it is left with only its name, since a job then has the job class's category.
|
|
108
|
+
#
|
|
109
|
+
# A category with a schema cannot have the options that the schema replaces, such as an input category's
|
|
110
|
+
# `allowed_columns`, see RocketJob::Category::Input.deprecated_column_options. Those that the form does not show are
|
|
111
|
+
# removed, since they could not be changed otherwise, so that giving a category a schema replaces them.
|
|
112
|
+
#
|
|
113
|
+
# A validate_rows of true that the form shows for a category that its job class already validates is removed, see
|
|
114
|
+
# .remove_inherited_validate_rows.
|
|
115
|
+
def self.merge_category(properties, posted, changes, job_class)
|
|
116
|
+
key = posted.property.to_s
|
|
117
|
+
categories = Array(properties[key]).dup
|
|
118
|
+
change = Array(changes[key]).find { |category| category["name"].to_s == posted.name } || {}
|
|
119
|
+
index = categories.index { |category| category.is_a?(Hash) && (category["name"] || "main").to_s == posted.name }
|
|
120
|
+
category = index ? categories[index].dup : {"name" => posted.name}
|
|
121
|
+
|
|
122
|
+
fields = posted.fields.map(&:to_s)
|
|
123
|
+
fields.each { |field| merge_property(category, change, field) }
|
|
124
|
+
remove_options_replaced_by_schema(category, posted, job_class)
|
|
125
|
+
remove_inherited_validate_rows(category, posted, job_class)
|
|
126
|
+
|
|
127
|
+
if (category.keys - ["name"]).empty?
|
|
128
|
+
categories.delete_at(index) if index
|
|
129
|
+
elsif index
|
|
130
|
+
categories[index] = category
|
|
131
|
+
else
|
|
132
|
+
categories << category
|
|
133
|
+
end
|
|
134
|
+
categories.empty? ? properties.delete(key) : properties[key] = categories
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Removes the options that the schema of the supplied category replaces, see .merge_category: its own schema, or else
|
|
138
|
+
# the one that its job class gives it. Those that the form posted stay, such as an input category's `skip_unknown`,
|
|
139
|
+
# since it can change them.
|
|
140
|
+
def self.remove_options_replaced_by_schema(category, posted, job_class)
|
|
141
|
+
schema = category["schema"].presence || job_class_category(job_class, posted)&.schema
|
|
142
|
+
return if schema.blank?
|
|
143
|
+
|
|
144
|
+
replaced = posted.category_class.deprecated_column_options(category.merge("schema" => schema))
|
|
145
|
+
category.except!(*(replaced - posted.fields.map(&:to_s)))
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category of a new job of the supplied class
|
|
149
|
+
# that the supplied posted category changes, see JobSanitizer.posted_categories, or nil when it has none.
|
|
150
|
+
def self.job_class_category(job_class, posted)
|
|
151
|
+
job_class.new.public_send(posted.property).find { |category| category.name.to_s == posted.name }
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Removes the validate_rows of true that the form shows for an input category that validates its lines without it,
|
|
155
|
+
# see ApplicationHelper#validate_rows_select, when its job class with the fields that the form shows already
|
|
156
|
+
# validates them, such as a job class that defines the category in the deprecated :hash mode, so that the entry keeps
|
|
157
|
+
# following its job class. It is kept when the entry's own properties that the form does not show are what validate
|
|
158
|
+
# them, such as a saved :hash mode, or would stop it, such as a mode of their own.
|
|
159
|
+
def self.remove_inherited_validate_rows(category, posted, job_class)
|
|
160
|
+
return unless category["validate_rows"] == true && posted.fields.include?(:validate_rows)
|
|
161
|
+
|
|
162
|
+
without = category.except("validate_rows")
|
|
163
|
+
shown = without.slice("name", *posted.fields.map(&:to_s))
|
|
164
|
+
return unless validates_rows?(job_class, posted, shown) && validates_rows?(job_class, posted, without)
|
|
165
|
+
|
|
166
|
+
category.delete("validate_rows")
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Returns [true|false] whether a job of the supplied class, built with the supplied properties of the posted input
|
|
170
|
+
# category, validates the lines of each file that it uploads, see RocketJob::Category::Input#upload_validates_rows?.
|
|
171
|
+
def self.validates_rows?(job_class, posted, category)
|
|
172
|
+
job = job_class.from_properties(posted.property.to_s => [category])
|
|
173
|
+
job.input_category(posted.name.to_sym).upload_validates_rows?
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Sets the supplied property to its change, or removes it when the form did not change it from the default.
|
|
177
|
+
def self.merge_property(properties, changes, key)
|
|
178
|
+
if changes.key?(key)
|
|
179
|
+
properties[key] = changes[key]
|
|
180
|
+
else
|
|
181
|
+
properties.delete(key)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
private_class_method :merge_category, :remove_options_replaced_by_schema, :job_class_category,
|
|
186
|
+
:remove_inherited_validate_rows, :validates_rows?, :merge_property
|
|
70
187
|
end
|
|
71
188
|
end
|
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
module RocketJobMissionControl
|
|
2
2
|
module JobSanitizer
|
|
3
|
-
CATEGORIES_FIELDS = %i[id name format format_options schema
|
|
3
|
+
CATEGORIES_FIELDS = %i[id name format format_options schema validate_rows skip_unknown slice_size columns].freeze
|
|
4
4
|
|
|
5
5
|
# The category fields whose blank value means their default, nil, such as no `schema`, so that blanking them
|
|
6
6
|
# clears a value that was set.
|
|
7
|
-
CLEARABLE_CATEGORY_FIELDS = %i[schema].freeze
|
|
7
|
+
CLEARABLE_CATEGORY_FIELDS = %i[schema validate_rows].freeze
|
|
8
|
+
|
|
9
|
+
# The param that the job and Dirmon entry forms post each kind of category in, by the property that holds those
|
|
10
|
+
# categories, with their class.
|
|
11
|
+
CATEGORY_PARAMS = {
|
|
12
|
+
input_categories: [:input_categories_attributes, RocketJob::Category::Input],
|
|
13
|
+
output_categories: [:output_categories_attributes, RocketJob::Category::Output]
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
# A category that a form posted, see .posted_categories: the property that holds it, such as :input_categories,
|
|
17
|
+
# its class, its name, and the names of its fields that the form posted.
|
|
18
|
+
PostedCategory = Struct.new(:property, :category_class, :name, :fields)
|
|
8
19
|
|
|
9
20
|
# Returns [Hash] the permissible params for the specified job class, after sanitizing.
|
|
10
21
|
# Parameters
|
|
@@ -31,8 +42,8 @@ module RocketJobMissionControl
|
|
|
31
42
|
next unless field&.type
|
|
32
43
|
|
|
33
44
|
case field.type.name
|
|
34
|
-
when "String"
|
|
35
|
-
value.gsub("\r\n", "\n")
|
|
45
|
+
when "String", "Symbol", "Mongoid::StringifiedSymbol"
|
|
46
|
+
value = value.gsub("\r\n", "\n")
|
|
36
47
|
when "Hash"
|
|
37
48
|
begin
|
|
38
49
|
value = value.blank? ? nil : JSON.parse(value)
|
|
@@ -52,22 +63,43 @@ module RocketJobMissionControl
|
|
|
52
63
|
end
|
|
53
64
|
end
|
|
54
65
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
permissible_params[:input_categories] = categories unless categories == [{}]
|
|
58
|
-
end
|
|
66
|
+
CATEGORY_PARAMS.each_pair do |property, (param, category_class)|
|
|
67
|
+
next unless properties.key?(param)
|
|
59
68
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
permissible_params[:output_categories] = categories unless categories == [{}]
|
|
69
|
+
categories = sanitize_categories(properties[param], category_class, nil_blank, target)
|
|
70
|
+
permissible_params[property] = categories unless categories == [{}]
|
|
63
71
|
end
|
|
64
72
|
|
|
65
73
|
permissible_params
|
|
66
74
|
end
|
|
67
75
|
|
|
68
|
-
# Returns [Array<
|
|
69
|
-
#
|
|
70
|
-
def self.
|
|
76
|
+
# Returns [Array<Symbol>] the user editable fields of the supplied job class that the supplied properties of a form
|
|
77
|
+
# post, see .sanitize, also those posted blank, which .sanitize can leave out.
|
|
78
|
+
def self.posted_fields(properties, job_class)
|
|
79
|
+
job_class.user_editable_fields.select { |field_name| properties.key?(field_name) }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Returns [Array<PostedCategory>] each category that the supplied properties of a form post, see .sanitize, with
|
|
83
|
+
# the names of its fields that the form posted, see CATEGORIES_FIELDS, also those posted blank, which .sanitize can
|
|
84
|
+
# leave out.
|
|
85
|
+
def self.posted_categories(properties)
|
|
86
|
+
CATEGORY_PARAMS.flat_map do |property, (param, category_class)|
|
|
87
|
+
next [] unless properties.key?(param)
|
|
88
|
+
|
|
89
|
+
properties[param].each_pair.map do |_, category|
|
|
90
|
+
fields = (CATEGORIES_FIELDS - %i[id name]).select { |key| category.key?(key) }
|
|
91
|
+
PostedCategory.new(property, category_class, (category[:name].presence || "main").to_s, fields)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Returns [Array<Hash>] the permissible properties of each category of the supplied category class, see .sanitize
|
|
97
|
+
# for `nil_blank`, which only applies to the CLEARABLE_CATEGORY_FIELDS, and `target`, which gets the error of a
|
|
98
|
+
# schema that is not valid JSON.
|
|
99
|
+
#
|
|
100
|
+
# Each value is converted by its field in the category class, such as a Boolean or an Integer rather than the text
|
|
101
|
+
# that the form posts, since a Dirmon entry saves its properties as they are given.
|
|
102
|
+
def self.sanitize_categories(properties, category_class, nil_blank = true, target = nil)
|
|
71
103
|
categories = []
|
|
72
104
|
|
|
73
105
|
properties.each_pair do |_, category|
|
|
@@ -76,18 +108,20 @@ module RocketJobMissionControl
|
|
|
76
108
|
next unless category.key?(key)
|
|
77
109
|
|
|
78
110
|
value = category[key]
|
|
111
|
+
# A multi-select posts a blank entry, so that it is posted even with nothing selected.
|
|
112
|
+
value = value.compact_blank if key == :columns && value.is_a?(Array)
|
|
79
113
|
if value.blank?
|
|
80
114
|
hash[key] = nil if nil_blank && CLEARABLE_CATEGORY_FIELDS.include?(key)
|
|
81
115
|
next
|
|
82
116
|
end
|
|
83
|
-
next if (key == :columns) && value == [""]
|
|
84
117
|
|
|
85
118
|
value = JSON.parse(value) if key == :format_options
|
|
86
119
|
if key == :schema
|
|
87
120
|
value = sanitize_schema(value, target)
|
|
88
121
|
next if value.nil?
|
|
89
122
|
end
|
|
90
|
-
|
|
123
|
+
field = category_class.fields[key.to_s]
|
|
124
|
+
hash[key] = field ? field.mongoize(value) : value
|
|
91
125
|
end
|
|
92
126
|
categories << hash
|
|
93
127
|
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
<%# The schemas that Rocket Job proposes to replace the deprecated category options of a job or Dirmon entry. %>
|
|
2
|
+
<%# The owner answers none for a job that can no longer run, and one with the error when its categories cannot be read. %>
|
|
3
|
+
<% proposals = owner.schema_proposals %>
|
|
4
|
+
<% if proposals.present? %>
|
|
5
|
+
<%# A proposal without changes only says why, such as a job that has already uploaded its input. %>
|
|
6
|
+
<% changes = proposals.any? { |proposal| proposal.changes.present? } %>
|
|
7
|
+
<div class="card schema-proposals">
|
|
8
|
+
<div class="card-header">
|
|
9
|
+
<h3 class="h6 mb-0">Proposed Schema</h3>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<div class="card-body">
|
|
13
|
+
<% if changes %>
|
|
14
|
+
<p>
|
|
15
|
+
Rocket Job 8 removes the deprecated category options below. Rocket Job proposes these settings to replace
|
|
16
|
+
them. Review each one, read a recent file with it (<code>RocketJob::SchemaProposal#dry_run</code>), and apply it by hand.
|
|
17
|
+
</p>
|
|
18
|
+
<% end %>
|
|
19
|
+
|
|
20
|
+
<% proposals.each do |proposal| %>
|
|
21
|
+
<div class="schema-proposal mb-3">
|
|
22
|
+
<h4 class="h6"><%= proposal.where %></h4>
|
|
23
|
+
|
|
24
|
+
<% if proposal.error %>
|
|
25
|
+
<div class="alert alert-warning">Cannot translate: <%= proposal.error %></div>
|
|
26
|
+
<% end %>
|
|
27
|
+
|
|
28
|
+
<% if proposal.changes.present? %>
|
|
29
|
+
<%# The same label and value table as the other details, see base.css. %>
|
|
30
|
+
<div class='row'>
|
|
31
|
+
<div class='col-md-12'>
|
|
32
|
+
<table>
|
|
33
|
+
<% proposal.changes.each_pair do |name, value| %>
|
|
34
|
+
<tr>
|
|
35
|
+
<td><label><%= name.to_s.titleize %>:</label></td>
|
|
36
|
+
<td>
|
|
37
|
+
<% if value.nil? %>
|
|
38
|
+
remove
|
|
39
|
+
<% elsif value.is_a?(Hash) || value.is_a?(Array) %>
|
|
40
|
+
<%= render_json_tree(value) %>
|
|
41
|
+
<% else %>
|
|
42
|
+
<%= value %>
|
|
43
|
+
<% end %>
|
|
44
|
+
</td>
|
|
45
|
+
</tr>
|
|
46
|
+
<% end %>
|
|
47
|
+
</table>
|
|
48
|
+
</div>
|
|
49
|
+
</div>
|
|
50
|
+
<% end %>
|
|
51
|
+
|
|
52
|
+
<% if proposal.notes.present? %>
|
|
53
|
+
<ul>
|
|
54
|
+
<% proposal.notes.each do |note| %>
|
|
55
|
+
<li><%= note %></li>
|
|
56
|
+
<% end %>
|
|
57
|
+
</ul>
|
|
58
|
+
<% end %>
|
|
59
|
+
</div>
|
|
60
|
+
<% end %>
|
|
61
|
+
|
|
62
|
+
<% if changes %>
|
|
63
|
+
<details>
|
|
64
|
+
<summary>What a schema reads differently</summary>
|
|
65
|
+
<ul>
|
|
66
|
+
<% RocketJob::SchemaProposal::FILE_CHECKS.each do |check| %>
|
|
67
|
+
<li><%= check %></li>
|
|
68
|
+
<% end %>
|
|
69
|
+
</ul>
|
|
70
|
+
</details>
|
|
71
|
+
<% end %>
|
|
72
|
+
</div>
|
|
73
|
+
</div>
|
|
74
|
+
<% end %>
|
|
@@ -17,11 +17,11 @@
|
|
|
17
17
|
</tr>
|
|
18
18
|
<tr>
|
|
19
19
|
<td><label>Pattern:</label></td>
|
|
20
|
-
<td><%= @dirmon_entry.
|
|
20
|
+
<td><%= @dirmon_entry.pattern_display_name %></td>
|
|
21
21
|
</tr>
|
|
22
22
|
<tr>
|
|
23
23
|
<td><label>Archive Directory:</label></td>
|
|
24
|
-
<td><%= @dirmon_entry.
|
|
24
|
+
<td><%= @dirmon_entry.archive_directory_display_name %></td>
|
|
25
25
|
</tr>
|
|
26
26
|
<tr>
|
|
27
27
|
<td><label>ID:</label></td>
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
<% action ||= :create %>
|
|
2
|
+
<%# Built before the errors are shown, since properties that cannot build a job add one. %>
|
|
3
|
+
<% @job = dirmon_entry_job(@dirmon_entry) %>
|
|
2
4
|
|
|
3
5
|
<% if @dirmon_entry.errors.present? %>
|
|
4
6
|
<div class="alert alert-alert">Invalid Dirmon entry!</div>
|
|
@@ -38,8 +40,7 @@
|
|
|
38
40
|
</div>
|
|
39
41
|
</div>
|
|
40
42
|
|
|
41
|
-
<% if @
|
|
42
|
-
<% @job = @dirmon_entry.job_class.from_properties(@dirmon_entry.properties) %>
|
|
43
|
+
<% if @job %>
|
|
43
44
|
<%= f.fields_for :properties do |p| %>
|
|
44
45
|
<div class="card">
|
|
45
46
|
<div class="card-header">
|
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
<div class="mb-3">
|
|
17
17
|
<%= f.label :columns %>
|
|
18
18
|
<% options = input_category.columns || []%>
|
|
19
|
-
|
|
19
|
+
<%# include_hidden posts the columns even when none are selected, so that clearing them removes them. %>
|
|
20
|
+
<%= f.select(:columns, options_for_select(options, options), { include_hidden: true }, { class: "tom-select form-select", multiple: true }) %>
|
|
20
21
|
</div>
|
|
21
22
|
|
|
22
23
|
<div class="mb-3">
|
|
@@ -38,10 +39,9 @@
|
|
|
38
39
|
</div>
|
|
39
40
|
|
|
40
41
|
<div class="mb-3">
|
|
41
|
-
<%= f.label :
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
<%= f.select(:mode, options, { include_blank: true, selected: value }, { class: "tom-select form-select" }) %>
|
|
42
|
+
<%= f.label :validate_rows %>
|
|
43
|
+
<%= validate_rows_select(f, input_category) %>
|
|
44
|
+
<div class="form-text">Parse each row of a file in the format while it is uploaded, so that a file with a row that cannot be read is rejected before processing starts. Needs a format.</div>
|
|
45
45
|
</div>
|
|
46
46
|
|
|
47
47
|
<div class="mb-3">
|
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
<div class="mb-3">
|
|
17
17
|
<%= f.label :columns %>
|
|
18
18
|
<% options = output_category.columns || [] %>
|
|
19
|
-
|
|
19
|
+
<%# include_hidden posts the columns even when none are selected, so that clearing them removes them. %>
|
|
20
|
+
<%= f.select(:columns, options_for_select(options, options), { include_hidden: true }, { class: "tom-select form-select", multiple: true }) %>
|
|
20
21
|
</div>
|
|
21
22
|
|
|
22
23
|
<div class="mb-3">
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
<div class='row'>
|
|
3
3
|
<div class='col-md-12'>
|
|
4
4
|
<table>
|
|
5
|
-
<% @dirmon_entry.
|
|
5
|
+
<% properties = @dirmon_entry.display_properties %>
|
|
6
|
+
<% properties.keys.sort.each do |key| %>
|
|
6
7
|
<%
|
|
7
|
-
value =
|
|
8
|
+
value = properties[key]
|
|
8
9
|
key = key.to_s.titleize
|
|
9
10
|
%>
|
|
10
11
|
<tr>
|
|
@@ -35,3 +35,4 @@
|
|
|
35
35
|
<%= render partial: 'details', locals: {dirmon_entry: @dirmon_entry} %>
|
|
36
36
|
<%= render partial: 'properties', locals: {dirmon_entry: @dirmon_entry} %>
|
|
37
37
|
<%= render partial: 'exception', locals: {dirmon_entry: @dirmon_entry} if @dirmon_entry.exception %>
|
|
38
|
+
<%= render partial: 'schema_proposals', locals: {owner: @dirmon_entry} %>
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
<% if category.file_name %>
|
|
16
16
|
<tr>
|
|
17
17
|
<td><label>File Name:</label></td>
|
|
18
|
-
<td><%= category
|
|
18
|
+
<td><%= job_category_file_name(category) %></td>
|
|
19
19
|
</tr>
|
|
20
20
|
<% end %>
|
|
21
21
|
</tr>
|
|
@@ -52,10 +52,10 @@
|
|
|
52
52
|
<td><%= render_json_tree(category.required_columns) %></td>
|
|
53
53
|
</tr>
|
|
54
54
|
<% end %>
|
|
55
|
-
<% if category.
|
|
55
|
+
<% if category.upload_validates_rows? %>
|
|
56
56
|
<tr>
|
|
57
|
-
<td><label>
|
|
58
|
-
<td
|
|
57
|
+
<td><label>Validate Rows:</label></td>
|
|
58
|
+
<td>true</td>
|
|
59
59
|
</tr>
|
|
60
60
|
<% end %>
|
|
61
61
|
<% if category.format_options %>
|
|
@@ -46,10 +46,9 @@
|
|
|
46
46
|
</div>
|
|
47
47
|
|
|
48
48
|
<div class="mb-3">
|
|
49
|
-
<%= f.label :
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
<%= f.select(:mode, options, { include_blank: true, selected: value }, { class: "tom-select form-select" }) %>
|
|
49
|
+
<%= f.label :validate_rows %>
|
|
50
|
+
<%= validate_rows_select(f, input_category) %>
|
|
51
|
+
<div class="form-text">Parse each row of a file in the format while it is uploaded, so that a file with a row that cannot be read is rejected before processing starts. Needs a format.</div>
|
|
53
52
|
</div>
|
|
54
53
|
|
|
55
54
|
<div class="mb-3">
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rocketjob_mission_control
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 7.
|
|
4
|
+
version: 7.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michael Cloutier
|
|
@@ -46,14 +46,14 @@ dependencies:
|
|
|
46
46
|
requirements:
|
|
47
47
|
- - "~>"
|
|
48
48
|
- !ruby/object:Gem::Version
|
|
49
|
-
version: '7.
|
|
49
|
+
version: '7.2'
|
|
50
50
|
type: :runtime
|
|
51
51
|
prerelease: false
|
|
52
52
|
version_requirements: !ruby/object:Gem::Requirement
|
|
53
53
|
requirements:
|
|
54
54
|
- - "~>"
|
|
55
55
|
- !ruby/object:Gem::Version
|
|
56
|
-
version: '7.
|
|
56
|
+
version: '7.2'
|
|
57
57
|
description: Rocket Job Mission Control is the Web user interface to manage Rocket
|
|
58
58
|
Job.
|
|
59
59
|
executables: []
|
|
@@ -130,6 +130,7 @@ files:
|
|
|
130
130
|
- app/views/layouts/rocket_job_mission_control/partials/_header.html.erb
|
|
131
131
|
- app/views/layouts/rocket_job_mission_control/partials/_sidebar.html.erb
|
|
132
132
|
- app/views/rocket_job_mission_control/active_workers/index.html.erb
|
|
133
|
+
- app/views/rocket_job_mission_control/application/_schema_proposals.html.erb
|
|
133
134
|
- app/views/rocket_job_mission_control/dirmon_entries/_details.html.erb
|
|
134
135
|
- app/views/rocket_job_mission_control/dirmon_entries/_exception.html.erb
|
|
135
136
|
- app/views/rocket_job_mission_control/dirmon_entries/_form.html.erb
|
|
@@ -180,7 +181,7 @@ metadata:
|
|
|
180
181
|
changelog_uri: https://github.com/reidmorrison/rocketjob_mission_control/blob/main/CHANGELOG.md
|
|
181
182
|
documentation_uri: https://rocketjob.reidmorrison.com/
|
|
182
183
|
homepage_uri: https://rocketjob.reidmorrison.com/
|
|
183
|
-
source_code_uri: https://github.com/reidmorrison/rocketjob_mission_control/tree/v7.
|
|
184
|
+
source_code_uri: https://github.com/reidmorrison/rocketjob_mission_control/tree/v7.2.0
|
|
184
185
|
rubygems_mfa_required: 'true'
|
|
185
186
|
rdoc_options: []
|
|
186
187
|
require_paths:
|