rocketjob_mission_control 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/rocket_job_mission_control/application_controller.rb +5 -0
- data/app/controllers/rocket_job_mission_control/dirmon_entries_controller.rb +18 -10
- data/app/controllers/rocket_job_mission_control/jobs_controller.rb +21 -21
- data/app/datatables/rocket_job_mission_control/dirmon_entries_datatable.rb +1 -1
- data/app/datatables/rocket_job_mission_control/servers_datatable.rb +1 -1
- data/app/helpers/rocket_job_mission_control/application_helper.rb +18 -32
- data/app/helpers/rocket_job_mission_control/dirmon_entries_helper.rb +16 -0
- data/app/helpers/rocket_job_mission_control/jobs_helper.rb +26 -6
- data/app/models/rocket_job_mission_control/dirmon_sanitizer.rb +121 -3
- data/app/models/rocket_job_mission_control/job_sanitizer.rb +76 -15
- data/app/models/rocket_job_mission_control/record_editor.rb +36 -0
- data/app/views/rocket_job_mission_control/application/_schema_proposals.html.erb +74 -0
- data/app/views/rocket_job_mission_control/dirmon_entries/_details.html.erb +2 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/_form.html.erb +3 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/_input_category_fields.html.erb +11 -6
- data/app/views/rocket_job_mission_control/dirmon_entries/_output_category_fields.html.erb +8 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/_properties.html.erb +3 -2
- data/app/views/rocket_job_mission_control/dirmon_entries/show.html.erb +1 -0
- data/app/views/rocket_job_mission_control/jobs/_details.html.erb +1 -1
- data/app/views/rocket_job_mission_control/jobs/_input_categories.html.erb +10 -4
- data/app/views/rocket_job_mission_control/jobs/_input_category_fields.html.erb +17 -5
- data/app/views/rocket_job_mission_control/jobs/_output_categories.html.erb +7 -1
- data/app/views/rocket_job_mission_control/jobs/_output_category_fields.html.erb +14 -1
- data/app/views/rocket_job_mission_control/jobs/edit_slice.html.erb +10 -10
- data/app/views/rocket_job_mission_control/jobs/show.html.erb +1 -0
- data/lib/rocket_job_mission_control/version.rb +1 -1
- metadata +6 -7
- data/app/assets/javascripts/rocket_job_mission_control/nested_fields.js +0 -112
- data/app/views/rocket_job_mission_control/dirmon_entries/_input_categories.html.erb +0 -59
- data/app/views/rocket_job_mission_control/dirmon_entries/_output_categories.html.erb +0 -44
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6fb6310a1a33fad99189a35c059a32f6eb77222f27276ca490eb80d61230e1c0
|
|
4
|
+
data.tar.gz: 316fb96d05902f8399717fc0cec7f7987c86906096ad209d40b297d2aad8e90d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 427be2abefed3b1eb0845b1fd26dd6258d4dbe06cc43c1cb23f4a5c2ff43d8192b9a6eb394027fd5ea43616543f23f62b2d88f5ff0f36d6c572a0a50bff23b2f
|
|
7
|
+
data.tar.gz: 074d0349f4df4001889ff54809005d5e2fccce394ada49ce4d534147e7c2ca8fb6fdfd3c54500d194fa6672d29629d2eb582d7607996a60ebea7388613403a30
|
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Rocket Job Mission Control
|
|
2
|
-
[](https://rubygems.org/gems/rocketjob_mission_control) [](https://github.com/reidmorrison/rocketjob_mission_control/actions?query=workflow%3Abuild) [](https://rubygems.org/gems/rocketjob_mission_control) [](http://opensource.org/licenses/Apache-2.0) 
|
|
2
|
+
[](https://rubygems.org/gems/rocketjob_mission_control) [](https://github.com/reidmorrison/rocketjob_mission_control/actions?query=workflow%3Abuild) [](https://rubygems.org/gems/rocketjob_mission_control) [](https://github.com/reidmorrison/rocketjob_mission_control/actions/workflows/ci.yml) [](http://opensource.org/licenses/Apache-2.0) 
|
|
3
3
|
|
|
4
4
|
Rocket Job Mission Control is the web based user interface for [Rocket Job](https://rocketjob.reidmorrison.com/),
|
|
5
5
|
Ruby's missing batch system.
|
|
@@ -2,6 +2,11 @@ module RocketJobMissionControl
|
|
|
2
2
|
class ApplicationController < ActionController::Base
|
|
3
3
|
protect_from_forgery with: :exception
|
|
4
4
|
|
|
5
|
+
# The status of a form that is rendered again to show why it could not be saved. Turbo Drive ignores a form
|
|
6
|
+
# response that renders with 200, so the errors would never be shown. Rack 2.2, which Rails 7.2 allows, does not
|
|
7
|
+
# know the name :unprocessable_content, and Rack 3.1 deprecates :unprocessable_entity.
|
|
8
|
+
FORM_ERROR_STATUS = 422
|
|
9
|
+
|
|
5
10
|
around_action :with_time_zone
|
|
6
11
|
|
|
7
12
|
private
|
|
@@ -62,21 +62,22 @@ module RocketJobMissionControl
|
|
|
62
62
|
if @dirmon_entry.errors.empty? && @dirmon_entry.save
|
|
63
63
|
redirect_to(dirmon_entry_path(@dirmon_entry))
|
|
64
64
|
else
|
|
65
|
-
render :new
|
|
65
|
+
render :new, status: FORM_ERROR_STATUS
|
|
66
66
|
end
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
def update
|
|
70
70
|
authorize! :update, @dirmon_entry
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
entry_params = params[:rocket_job_dirmon_entry]
|
|
72
|
+
sanitized_params = DirmonSanitizer.sanitize(entry_params, @dirmon_entry.job_class, @dirmon_entry)
|
|
73
73
|
|
|
74
|
-
sanitized_params[:properties] =
|
|
74
|
+
sanitized_params[:properties] =
|
|
75
|
+
DirmonSanitizer.update_properties(entry_params[:properties], sanitized_params[:properties], @dirmon_entry)
|
|
75
76
|
|
|
76
77
|
if @dirmon_entry.errors.empty? && @dirmon_entry.valid? && @dirmon_entry.update(sanitized_params)
|
|
77
78
|
redirect_to(rocket_job_mission_control.dirmon_entry_path(@dirmon_entry))
|
|
78
79
|
else
|
|
79
|
-
render :edit
|
|
80
|
+
render :edit, status: FORM_ERROR_STATUS
|
|
80
81
|
end
|
|
81
82
|
end
|
|
82
83
|
|
|
@@ -96,11 +97,11 @@ module RocketJobMissionControl
|
|
|
96
97
|
# the replicate method clones the existing Dirmon Entity
|
|
97
98
|
def replicate
|
|
98
99
|
authorize! :replicate, @dirmon_entry
|
|
99
|
-
dirmon_entry_replicate =
|
|
100
|
+
dirmon_entry_replicate = @dirmon_entry.replicate
|
|
100
101
|
|
|
101
102
|
if (properties = params[:rocket_job_dirmon_entry][:properties])
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
sanitized = DirmonSanitizer.sanitize_properties(properties, dirmon_entry_replicate.job_class, @dirmon_entry)
|
|
104
|
+
dirmon_entry_replicate.properties = DirmonSanitizer.update_properties(properties, sanitized, @dirmon_entry)
|
|
104
105
|
end
|
|
105
106
|
|
|
106
107
|
if dirmon_entry_replicate.update(dirmon_params)
|
|
@@ -111,7 +112,7 @@ module RocketJobMissionControl
|
|
|
111
112
|
dirmon_entry_replicate.errors.messages.each_pair do |field, message|
|
|
112
113
|
@dirmon_entry.errors.add(field, message)
|
|
113
114
|
end
|
|
114
|
-
render :copy
|
|
115
|
+
render :copy, status: FORM_ERROR_STATUS
|
|
115
116
|
end
|
|
116
117
|
end
|
|
117
118
|
|
|
@@ -160,6 +161,13 @@ module RocketJobMissionControl
|
|
|
160
161
|
params.fetch(:rocket_job_dirmon_entry, {}).permit(:name, :archive_directory, :pattern, :job_class_name, :properties)
|
|
161
162
|
end
|
|
162
163
|
|
|
164
|
+
# The columns of the Dirmon entries list that its search box searches. A pattern can hold credentials, which the
|
|
165
|
+
# list leaves out, see RocketJob::DirmonEntry#pattern_display_name, so it is only searched for a role that can edit
|
|
166
|
+
# an entry, whose form shows the pattern as stored. Otherwise a search could find a password one letter at a time.
|
|
167
|
+
def dirmon_search_columns
|
|
168
|
+
can?(:edit, RocketJob::DirmonEntry) ? %i[job_class_name name pattern] : %i[job_class_name name]
|
|
169
|
+
end
|
|
170
|
+
|
|
163
171
|
def render_datatable(entries, description)
|
|
164
172
|
respond_to do |format|
|
|
165
173
|
format.html do
|
|
@@ -169,7 +177,7 @@ module RocketJobMissionControl
|
|
|
169
177
|
|
|
170
178
|
format.json do
|
|
171
179
|
query = RocketJobMissionControl::Query.new(entries, name: :asc)
|
|
172
|
-
query.search_columns =
|
|
180
|
+
query.search_columns = dirmon_search_columns
|
|
173
181
|
query.display_columns = %w[name _type pattern]
|
|
174
182
|
render(json: DirmonEntriesDatatable.new(view_context, query))
|
|
175
183
|
end
|
|
@@ -83,7 +83,7 @@ module RocketJobMissionControl
|
|
|
83
83
|
if @job.errors.empty? && @job.valid? && @job.save
|
|
84
84
|
redirect_to job_path(@job)
|
|
85
85
|
else
|
|
86
|
-
render :edit
|
|
86
|
+
render :edit, status: FORM_ERROR_STATUS
|
|
87
87
|
end
|
|
88
88
|
end
|
|
89
89
|
|
|
@@ -172,32 +172,32 @@ module RocketJobMissionControl
|
|
|
172
172
|
authorize! :update_slice, @job
|
|
173
173
|
|
|
174
174
|
# Params from the edit_slice form
|
|
175
|
-
error_type
|
|
176
|
-
offset
|
|
177
|
-
|
|
175
|
+
error_type = params[:error_type]
|
|
176
|
+
offset = params.fetch(:offset, 0).to_i
|
|
177
|
+
line_index = params[:line_index].to_i
|
|
178
178
|
|
|
179
179
|
# Find the same slice edit_slice displayed: filtered by error type and
|
|
180
180
|
# ordered so the offset selects the matching slice.
|
|
181
181
|
scope = @job.input.failed.where("exception.class_name" => error_type)
|
|
182
182
|
slice = scope.order(_id: 1).limit(1).skip(offset).first
|
|
183
183
|
|
|
184
|
-
#
|
|
185
|
-
#
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
slice.records = updated_records
|
|
184
|
+
# Replace only the edited record, converting its text back into a record
|
|
185
|
+
# of the same kind, see RecordEditor. The other records are left as they are.
|
|
186
|
+
records = slice.to_a.dup
|
|
187
|
+
records[line_index] = RecordEditor.from_text(params.dig("job", "record"), records[line_index])
|
|
188
|
+
slice.records = records
|
|
190
189
|
|
|
191
190
|
if slice.save
|
|
192
|
-
logger.info("Slice Updated By #{login}, job: #{@job.id}, file_name: #{@job
|
|
191
|
+
logger.info("Slice Updated By #{login}, job: #{@job.id}, file_name: #{helpers.job_upload_file_name(@job)}")
|
|
193
192
|
flash[:success] = "slice updated"
|
|
194
193
|
redirect_to view_slice_job_path(@job, error_type: error_type, offset: offset)
|
|
195
194
|
else
|
|
196
195
|
flash[:danger] = "Error updating slice."
|
|
197
196
|
end
|
|
198
|
-
rescue EncodingError => e
|
|
197
|
+
rescue EncodingError, JSON::ParserError, BSON::Error => e
|
|
199
198
|
# Mongo only stores UTF-8, so a record left with invalid bytes cannot be
|
|
200
|
-
# saved
|
|
199
|
+
# saved, and the JSON of a record that is not text must be valid.
|
|
200
|
+
# Surface it instead of returning a 500.
|
|
201
201
|
flash[:danger] = "Error updating slice: #{e.message}"
|
|
202
202
|
redirect_to view_slice_job_path(@job, error_type: params[:error_type], offset: offset)
|
|
203
203
|
end
|
|
@@ -214,15 +214,13 @@ module RocketJobMissionControl
|
|
|
214
214
|
scope = @job.input.failed.where("exception.class_name" => error_type)
|
|
215
215
|
slice = scope.order(_id: 1).limit(1).skip(offset).first
|
|
216
216
|
|
|
217
|
-
#
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
# Assings full array back to slice
|
|
222
|
-
slice.records = slice.to_a
|
|
217
|
+
# Deletes only the selected record, not every record equal to it.
|
|
218
|
+
records = slice.to_a.dup
|
|
219
|
+
records.delete_at(line_index)
|
|
220
|
+
slice.records = records
|
|
223
221
|
|
|
224
222
|
if slice.save
|
|
225
|
-
logger.info("Line Deleted By #{login}, job: #{@job.id}, file_name: #{@job
|
|
223
|
+
logger.info("Line Deleted By #{login}, job: #{@job.id}, file_name: #{helpers.job_upload_file_name(@job)}")
|
|
226
224
|
flash[:success] = "Record #{slice.first_record_number + line_index} removed from the slice."
|
|
227
225
|
redirect_to view_slice_job_path(@job, error_type: error_type, offset: offset), status: :see_other
|
|
228
226
|
else
|
|
@@ -301,7 +299,8 @@ module RocketJobMissionControl
|
|
|
301
299
|
:name,
|
|
302
300
|
:format,
|
|
303
301
|
:format_options,
|
|
304
|
-
:
|
|
302
|
+
:schema,
|
|
303
|
+
:validate_rows,
|
|
305
304
|
:skip_unknown,
|
|
306
305
|
:slice_size,
|
|
307
306
|
{columns: []}
|
|
@@ -311,6 +310,7 @@ module RocketJobMissionControl
|
|
|
311
310
|
:name,
|
|
312
311
|
:format,
|
|
313
312
|
:format_options,
|
|
313
|
+
:schema,
|
|
314
314
|
{columns: []}
|
|
315
315
|
]
|
|
316
316
|
)
|
|
@@ -8,7 +8,7 @@ module RocketJobMissionControl
|
|
|
8
8
|
{
|
|
9
9
|
"0" => name_with_link(dirmon),
|
|
10
10
|
"1" => h(dirmon.job_class_name),
|
|
11
|
-
"2" => h(dirmon.
|
|
11
|
+
"2" => h(dirmon.pattern_display_name.try(:truncate, 80)),
|
|
12
12
|
"DT_RowClass" => "rjmc-card callout callout-#{dirmon.state}"
|
|
13
13
|
}
|
|
14
14
|
end
|
|
@@ -62,7 +62,7 @@ module RocketJobMissionControl
|
|
|
62
62
|
|
|
63
63
|
if server.stopping? && view.can?(:destroy, server)
|
|
64
64
|
actions += "Server is stopping..."
|
|
65
|
-
confirmation = ""
|
|
65
|
+
confirmation = +""
|
|
66
66
|
unless server.zombie?
|
|
67
67
|
confirmation << "Warning!\n\nDestroying this server will hard kill its active workers/jobs.\nKilled jobs will be requeued for processing on another worker.\n\n"
|
|
68
68
|
end
|
|
@@ -83,6 +83,24 @@ module RocketJobMissionControl
|
|
|
83
83
|
values
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
+
# Returns the text area for the schema of a category, see RocketJob::Category::Base#schema: the JSON of a schema,
|
|
87
|
+
# or the path or url of the JSON file that holds it. Blank is no schema.
|
|
88
|
+
def schema_text_area(f, value)
|
|
89
|
+
text = value.is_a?(Hash) ? JSON.pretty_generate(value) : value
|
|
90
|
+
f.text_area(:schema, value: text, rows: 4, class: "form-control font-monospace",
|
|
91
|
+
placeholder: '{"fields": [{"name": "name"}, {"name": "age", "type": "integer"}]}')
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Returns the select for the validate_rows of an input category, see RocketJob::Category::Input#validate_rows:
|
|
95
|
+
# blank, which clears it on a job and leaves it to the job class on a Dirmon entry, true or false. A category without
|
|
96
|
+
# one whose upload validates its lines anyway, such as one with a schema in the deprecated :hash mode, shows true,
|
|
97
|
+
# as Rocket Job 8 reads it, so that the form shows what the category does.
|
|
98
|
+
def validate_rows_select(f, category)
|
|
99
|
+
value = category.validate_rows
|
|
100
|
+
value = true if value.nil? && category.upload_validates_rows?
|
|
101
|
+
f.select(:validate_rows, [true, false], {include_blank: true, selected: value}, {class: "tom-select form-select"})
|
|
102
|
+
end
|
|
103
|
+
|
|
86
104
|
# Returns the editable field as html for use in editing dynamic fields from a Job class.
|
|
87
105
|
def editable_field_html(klass, field_name, value, f)
|
|
88
106
|
# When editing a job the values are of the correct type.
|
|
@@ -123,37 +141,5 @@ placeholder: '{"key1":"value1", "key2":"value2", "key3":"value3"}')
|
|
|
123
141
|
f.text_field(field_name, value: value, class: "form-control", placeholder: placeholder)
|
|
124
142
|
end
|
|
125
143
|
end
|
|
126
|
-
|
|
127
|
-
# This method creates a link with `data-id` `data-fields` attributes. These attributes are used to create new instances of the nested fields through Javascript.
|
|
128
|
-
def link_to_add_fields(name, f, association, option)
|
|
129
|
-
# Takes an object (@job) and creates a new instance of its associated model (:properties)
|
|
130
|
-
new_object = f.object.send(association).klass.new
|
|
131
|
-
|
|
132
|
-
# Saves the unique ID of the object into a variable.
|
|
133
|
-
# This is needed to ensure the key of the associated array is unique. This is makes parsing the content in the `data-fields` attribute easier through Javascript.
|
|
134
|
-
# We could use another method to achive this.
|
|
135
|
-
id = new_object.object_id
|
|
136
|
-
|
|
137
|
-
# https://api.rubyonrails.org/ fields_for(record_name, record_object = nil, fields_options = {}, &block)
|
|
138
|
-
# record_name = :addresses
|
|
139
|
-
# record_object = new_object
|
|
140
|
-
# fields_options = { child_index: id }
|
|
141
|
-
# child_index` is used to ensure the key of the associated array is unique, and that it matched the value in the `data-id` attribute.
|
|
142
|
-
# `person[addresses_attributes][child_index_value][_destroy]`
|
|
143
|
-
fields = f.fields_for(association, new_object, child_index: id) do |builder|
|
|
144
|
-
# `association.to_s.singularize + "_fields"` ends up evaluating to `address_fields`
|
|
145
|
-
# The render function will then look for `views/people/_address_fields.html.erb`
|
|
146
|
-
# The render function also needs to be passed the value of 'builder', because `views/dirmon_entries/_input_categories.html.erb` needs this to render the form tags.
|
|
147
|
-
render("#{association.to_s.singularize}_fields", f: builder)
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
# This renders a simple link, but passes information into `data` attributes.
|
|
151
|
-
# This info can be named anything we want, but in this case we chose `data-id:` and `data-fields:`.
|
|
152
|
-
# The `id:` is from `new_object.object_id`.
|
|
153
|
-
# The `fields:` are rendered from the `fields` blocks.
|
|
154
|
-
# We use `gsub("\n", "")` to remove anywhite space from the rendered partial.
|
|
155
|
-
# The `id:` value needs to match the value used in `child_index: id`.
|
|
156
|
-
link_to(name, "#", class: "add_fields btn btn-#{option}", data: {id: id, fields: fields.delete("\n")})
|
|
157
|
-
end
|
|
158
144
|
end
|
|
159
145
|
end
|
|
@@ -11,6 +11,22 @@ module RocketJobMissionControl
|
|
|
11
11
|
nil
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
# Returns [RocketJob::Job] the job that the entry's properties build, whose values its form shows, or nil without a
|
|
15
|
+
# job class. Returns nil too when Rocket Job cannot build one from them, such as for a category that the job class
|
|
16
|
+
# does not define or an option that it no longer has, and adds why to the entry's errors, so that its form still
|
|
17
|
+
# shows its other attributes rather than failing. The error is logged with its backtrace too, since it can come from
|
|
18
|
+
# the job class itself, such as a mistake in a default of one of its fields.
|
|
19
|
+
def dirmon_entry_job(dirmon_entry)
|
|
20
|
+
dirmon_entry.job_class&.from_properties(dirmon_entry.properties)
|
|
21
|
+
rescue ArgumentError, NoMethodError, Mongoid::Errors::MongoidError => e
|
|
22
|
+
job_class_name = dirmon_entry.job_class_name
|
|
23
|
+
logger.warn("Dirmon entry #{dirmon_entry.id}: its properties cannot build a #{job_class_name}: #{e.class}: " \
|
|
24
|
+
"#{e.message}\n#{Array(e.backtrace).join("\n")}")
|
|
25
|
+
reason = e.respond_to?(:problem) ? e.problem : e.message
|
|
26
|
+
dirmon_entry.errors.add(:properties, "cannot build a #{job_class_name}, so they cannot be edited here: #{reason}")
|
|
27
|
+
nil
|
|
28
|
+
end
|
|
29
|
+
|
|
14
30
|
def rocket_job_mission_control
|
|
15
31
|
@@rocket_job_mission_control_engine_url_helpers ||= RocketJobMissionControl::Engine.routes.url_helpers
|
|
16
32
|
end
|
|
@@ -107,8 +107,27 @@ module RocketJobMissionControl
|
|
|
107
107
|
(job.record_count.to_f / job.input_category.slice_size).ceil
|
|
108
108
|
end
|
|
109
109
|
|
|
110
|
+
# The job's upload file name, without any credentials, such as the password of an SFTP url that Dirmon
|
|
111
|
+
# archived the file to, see RocketJob.path_display_name.
|
|
112
|
+
def job_upload_file_name(job)
|
|
113
|
+
RocketJob.path_display_name(job.upload_file_name)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# The category's file name, without any credentials, such as the password of an SFTP url that the output is
|
|
117
|
+
# downloaded to. The category decides how its file name is shown, see RocketJob::Category::Base.display_properties.
|
|
118
|
+
def job_category_file_name(category)
|
|
119
|
+
category.class.display_properties("file_name" => category.file_name)["file_name"]
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The category's schema: its fields as a JSON tree, or the path or url of the JSON file that holds it, without any
|
|
123
|
+
# credentials. The category decides how its schema is shown, see RocketJob::Category::Base.display_properties.
|
|
124
|
+
def job_category_schema(category)
|
|
125
|
+
schema = category.class.display_properties("schema" => category.schema)["schema"]
|
|
126
|
+
schema.is_a?(Hash) ? render_json_tree(schema) : schema
|
|
127
|
+
end
|
|
128
|
+
|
|
110
129
|
def job_custom_fields(job)
|
|
111
|
-
attrs = job.
|
|
130
|
+
attrs = job.display_attributes
|
|
112
131
|
DISPLAYED_FIELDS.each { |key| attrs.delete(key) }
|
|
113
132
|
# Convert time zones for any custom time fields
|
|
114
133
|
attrs.keys { |key| attrs[key] = attrs[key].in_time_zone(Time.zone) if attrs[key].is_a?(Time) }
|
|
@@ -183,19 +202,20 @@ module RocketJobMissionControl
|
|
|
183
202
|
backtrace_strip_paths.any? { |path| line.to_s.start_with?(path) }
|
|
184
203
|
end
|
|
185
204
|
|
|
186
|
-
# Reversible
|
|
187
|
-
# See
|
|
205
|
+
# Reversible form of a failed record for an edit textarea, which keeps its type.
|
|
206
|
+
# See RecordEditor; convert it back with RecordEditor.from_text on save.
|
|
188
207
|
def escape_record(value)
|
|
189
|
-
|
|
208
|
+
RecordEditor.to_text(value)
|
|
190
209
|
end
|
|
191
210
|
|
|
192
211
|
# Read-only HTML for a failed record, with each unprintable/invalid byte
|
|
193
212
|
# rendered as a highlighted `\xHH` token. Each highlight carries a hover
|
|
194
213
|
# tooltip naming the byte, so an operator can see (in context) exactly which
|
|
195
|
-
# byte broke the record and what it is.
|
|
214
|
+
# byte broke the record and what it is. A record that is not a String, such as
|
|
215
|
+
# a Hash, is shown as Extended JSON, see RecordEditor.
|
|
196
216
|
def highlight_record(value)
|
|
197
217
|
safe_join(
|
|
198
|
-
RecordEscaper.segments(value).map do |type, text|
|
|
218
|
+
RecordEscaper.segments(RecordEditor.to_display_text(value)).map do |type, text|
|
|
199
219
|
next text if type == :text
|
|
200
220
|
|
|
201
221
|
content_tag(:span, text, class: "record-escape", title: record_escape_title(text))
|
|
@@ -13,17 +13,52 @@ module RocketJobMissionControl
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
if params.key?(:properties)
|
|
16
|
-
properties =
|
|
16
|
+
properties = sanitize_properties(params[:properties], job_class, target)
|
|
17
17
|
permissible_params[:properties] = properties if properties.present?
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
permissible_params
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# Returns [Hash] the supplied properties that a form posted, sanitized for a Dirmon entry, see JobSanitizer.sanitize,
|
|
24
|
+
# without those posted blank, which .update_properties removes from the entry.
|
|
25
|
+
def self.sanitize_properties(posted, job_class, target)
|
|
26
|
+
JobSanitizer.sanitize(posted, job_class, target, false)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Returns [Hash] the properties of the supplied Dirmon entry with the changes that its form posted, see
|
|
30
|
+
# .diff_properties: each property that the form posted is set when it differs from the job class's default, and
|
|
31
|
+
# removed otherwise, so that the job class decides it. Every other property is kept as it is saved, such as an
|
|
32
|
+
# input category's `allowed_columns` or `file_name`, since the form does not show it.
|
|
33
|
+
#
|
|
34
|
+
# Parameters
|
|
35
|
+
# posted [Hash]
|
|
36
|
+
# The properties that the form posted, see JobSanitizer.posted_fields and .posted_categories.
|
|
37
|
+
#
|
|
38
|
+
# sanitized [Hash]
|
|
39
|
+
# The same properties after .sanitize_properties.
|
|
40
|
+
#
|
|
41
|
+
# dirmon_entry [RocketJob::DirmonEntry]
|
|
42
|
+
# The entry whose properties the form shows, such as the one being edited or copied.
|
|
43
|
+
def self.update_properties(posted, sanitized, dirmon_entry)
|
|
44
|
+
properties = dirmon_entry.properties.to_h.deep_stringify_keys
|
|
45
|
+
return properties if posted.blank?
|
|
46
|
+
|
|
47
|
+
changes = diff_properties(sanitized, dirmon_entry).deep_stringify_keys
|
|
48
|
+
JobSanitizer.posted_fields(posted, dirmon_entry.job_class).each do |name|
|
|
49
|
+
merge_property(properties, changes, name.to_s)
|
|
50
|
+
end
|
|
51
|
+
JobSanitizer.posted_categories(posted).each do |category|
|
|
52
|
+
merge_category(properties, category, changes, dirmon_entry.job_class)
|
|
53
|
+
end
|
|
54
|
+
properties
|
|
55
|
+
end
|
|
56
|
+
|
|
23
57
|
# Returns [Hash] the difference between the supplied params and those already set in the job itself
|
|
24
58
|
def self.diff_properties(sanitized_properties, dirmon_entry)
|
|
25
59
|
default_job = dirmon_entry.job_class.new
|
|
26
|
-
|
|
60
|
+
# Without properties to change, such as when only the pattern changed, the job has the defaults.
|
|
61
|
+
updated_job = dirmon_entry.job_class.from_properties(sanitized_properties || {})
|
|
27
62
|
properties = {}
|
|
28
63
|
sanitized_properties&.each_pair do |name, value|
|
|
29
64
|
if name == :input_categories
|
|
@@ -45,7 +80,7 @@ module RocketJobMissionControl
|
|
|
45
80
|
end
|
|
46
81
|
properties[:output_categories] = categories unless categories.empty?
|
|
47
82
|
elsif default_job.public_send(name) != updated_job.public_send(name)
|
|
48
|
-
properties[name] = value
|
|
83
|
+
properties[name] = value
|
|
49
84
|
end
|
|
50
85
|
end
|
|
51
86
|
properties
|
|
@@ -66,5 +101,88 @@ module RocketJobMissionControl
|
|
|
66
101
|
diff[:name] = name unless diff.empty?
|
|
67
102
|
diff
|
|
68
103
|
end
|
|
104
|
+
|
|
105
|
+
# Merges the changes that the form posted for the supplied category, see JobSanitizer.posted_categories, into the
|
|
106
|
+
# supplied properties, see .update_properties. A saved category is found by its name, or is main without one, as a
|
|
107
|
+
# job reads it, and is removed when it is left with only its name, since a job then has the job class's category.
|
|
108
|
+
#
|
|
109
|
+
# A category with a schema cannot have the options that the schema replaces, such as an input category's
|
|
110
|
+
# `allowed_columns`, see RocketJob::Category::Input.deprecated_column_options. Those that the form does not show are
|
|
111
|
+
# removed, since they could not be changed otherwise, so that giving a category a schema replaces them.
|
|
112
|
+
#
|
|
113
|
+
# A validate_rows of true that the form shows for a category that its job class already validates is removed, see
|
|
114
|
+
# .remove_inherited_validate_rows.
|
|
115
|
+
def self.merge_category(properties, posted, changes, job_class)
|
|
116
|
+
key = posted.property.to_s
|
|
117
|
+
categories = Array(properties[key]).dup
|
|
118
|
+
change = Array(changes[key]).find { |category| category["name"].to_s == posted.name } || {}
|
|
119
|
+
index = categories.index { |category| category.is_a?(Hash) && (category["name"] || "main").to_s == posted.name }
|
|
120
|
+
category = index ? categories[index].dup : {"name" => posted.name}
|
|
121
|
+
|
|
122
|
+
fields = posted.fields.map(&:to_s)
|
|
123
|
+
fields.each { |field| merge_property(category, change, field) }
|
|
124
|
+
remove_options_replaced_by_schema(category, posted, job_class)
|
|
125
|
+
remove_inherited_validate_rows(category, posted, job_class)
|
|
126
|
+
|
|
127
|
+
if (category.keys - ["name"]).empty?
|
|
128
|
+
categories.delete_at(index) if index
|
|
129
|
+
elsif index
|
|
130
|
+
categories[index] = category
|
|
131
|
+
else
|
|
132
|
+
categories << category
|
|
133
|
+
end
|
|
134
|
+
categories.empty? ? properties.delete(key) : properties[key] = categories
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Removes the options that the schema of the supplied category replaces, see .merge_category: its own schema, or else
|
|
138
|
+
# the one that its job class gives it. Those that the form posted stay, such as an input category's `skip_unknown`,
|
|
139
|
+
# since it can change them.
|
|
140
|
+
def self.remove_options_replaced_by_schema(category, posted, job_class)
|
|
141
|
+
schema = category["schema"].presence || job_class_category(job_class, posted)&.schema
|
|
142
|
+
return if schema.blank?
|
|
143
|
+
|
|
144
|
+
replaced = posted.category_class.deprecated_column_options(category.merge("schema" => schema))
|
|
145
|
+
category.except!(*(replaced - posted.fields.map(&:to_s)))
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category of a new job of the supplied class
|
|
149
|
+
# that the supplied posted category changes, see JobSanitizer.posted_categories, or nil when it has none.
|
|
150
|
+
def self.job_class_category(job_class, posted)
|
|
151
|
+
job_class.new.public_send(posted.property).find { |category| category.name.to_s == posted.name }
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Removes the validate_rows of true that the form shows for an input category that validates its lines without it,
|
|
155
|
+
# see ApplicationHelper#validate_rows_select, when its job class with the fields that the form shows already
|
|
156
|
+
# validates them, such as a job class that defines the category in the deprecated :hash mode, so that the entry keeps
|
|
157
|
+
# following its job class. It is kept when the entry's own properties that the form does not show are what validate
|
|
158
|
+
# them, such as a saved :hash mode, or would stop it, such as a mode of their own.
|
|
159
|
+
def self.remove_inherited_validate_rows(category, posted, job_class)
|
|
160
|
+
return unless category["validate_rows"] == true && posted.fields.include?(:validate_rows)
|
|
161
|
+
|
|
162
|
+
without = category.except("validate_rows")
|
|
163
|
+
shown = without.slice("name", *posted.fields.map(&:to_s))
|
|
164
|
+
return unless validates_rows?(job_class, posted, shown) && validates_rows?(job_class, posted, without)
|
|
165
|
+
|
|
166
|
+
category.delete("validate_rows")
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Returns [true|false] whether a job of the supplied class, built with the supplied properties of the posted input
|
|
170
|
+
# category, validates the lines of each file that it uploads, see RocketJob::Category::Input#upload_validates_rows?.
|
|
171
|
+
def self.validates_rows?(job_class, posted, category)
|
|
172
|
+
job = job_class.from_properties(posted.property.to_s => [category])
|
|
173
|
+
job.input_category(posted.name.to_sym).upload_validates_rows?
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Sets the supplied property to its change, or removes it when the form did not change it from the default.
|
|
177
|
+
def self.merge_property(properties, changes, key)
|
|
178
|
+
if changes.key?(key)
|
|
179
|
+
properties[key] = changes[key]
|
|
180
|
+
else
|
|
181
|
+
properties.delete(key)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
private_class_method :merge_category, :remove_options_replaced_by_schema, :job_class_category,
|
|
186
|
+
:remove_inherited_validate_rows, :validates_rows?, :merge_property
|
|
69
187
|
end
|
|
70
188
|
end
|
|
@@ -1,6 +1,21 @@
|
|
|
1
1
|
module RocketJobMissionControl
|
|
2
2
|
module JobSanitizer
|
|
3
|
-
CATEGORIES_FIELDS = %i[id name format format_options
|
|
3
|
+
CATEGORIES_FIELDS = %i[id name format format_options schema validate_rows skip_unknown slice_size columns].freeze
|
|
4
|
+
|
|
5
|
+
# The category fields whose blank value means their default, nil, such as no `schema`, so that blanking them
|
|
6
|
+
# clears a value that was set.
|
|
7
|
+
CLEARABLE_CATEGORY_FIELDS = %i[schema validate_rows].freeze
|
|
8
|
+
|
|
9
|
+
# The param that the job and Dirmon entry forms post each kind of category in, by the property that holds those
|
|
10
|
+
# categories, with their class.
|
|
11
|
+
CATEGORY_PARAMS = {
|
|
12
|
+
input_categories: [:input_categories_attributes, RocketJob::Category::Input],
|
|
13
|
+
output_categories: [:output_categories_attributes, RocketJob::Category::Output]
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
# A category that a form posted, see .posted_categories: the property that holds it, such as :input_categories,
|
|
17
|
+
# its class, its name, and the names of its fields that the form posted.
|
|
18
|
+
PostedCategory = Struct.new(:property, :category_class, :name, :fields)
|
|
4
19
|
|
|
5
20
|
# Returns [Hash] the permissible params for the specified job class, after sanitizing.
|
|
6
21
|
# Parameters
|
|
@@ -27,8 +42,8 @@ module RocketJobMissionControl
|
|
|
27
42
|
next unless field&.type
|
|
28
43
|
|
|
29
44
|
case field.type.name
|
|
30
|
-
when "String"
|
|
31
|
-
value.gsub("\r\n", "\n")
|
|
45
|
+
when "String", "Symbol", "Mongoid::StringifiedSymbol"
|
|
46
|
+
value = value.gsub("\r\n", "\n")
|
|
32
47
|
when "Hash"
|
|
33
48
|
begin
|
|
34
49
|
value = value.blank? ? nil : JSON.parse(value)
|
|
@@ -48,20 +63,43 @@ module RocketJobMissionControl
|
|
|
48
63
|
end
|
|
49
64
|
end
|
|
50
65
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
permissible_params[:input_categories] = categories unless categories == [{}]
|
|
54
|
-
end
|
|
66
|
+
CATEGORY_PARAMS.each_pair do |property, (param, category_class)|
|
|
67
|
+
next unless properties.key?(param)
|
|
55
68
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
permissible_params[:output_categories] = categories unless categories == [{}]
|
|
69
|
+
categories = sanitize_categories(properties[param], category_class, nil_blank, target)
|
|
70
|
+
permissible_params[property] = categories unless categories == [{}]
|
|
59
71
|
end
|
|
60
72
|
|
|
61
73
|
permissible_params
|
|
62
74
|
end
|
|
63
75
|
|
|
64
|
-
|
|
76
|
+
# Returns [Array<Symbol>] the user editable fields of the supplied job class that the supplied properties of a form
|
|
77
|
+
# post, see .sanitize, also those posted blank, which .sanitize can leave out.
|
|
78
|
+
def self.posted_fields(properties, job_class)
|
|
79
|
+
job_class.user_editable_fields.select { |field_name| properties.key?(field_name) }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Returns [Array<PostedCategory>] each category that the supplied properties of a form post, see .sanitize, with
|
|
83
|
+
# the names of its fields that the form posted, see CATEGORIES_FIELDS, also those posted blank, which .sanitize can
|
|
84
|
+
# leave out.
|
|
85
|
+
def self.posted_categories(properties)
|
|
86
|
+
CATEGORY_PARAMS.flat_map do |property, (param, category_class)|
|
|
87
|
+
next [] unless properties.key?(param)
|
|
88
|
+
|
|
89
|
+
properties[param].each_pair.map do |_, category|
|
|
90
|
+
fields = (CATEGORIES_FIELDS - %i[id name]).select { |key| category.key?(key) }
|
|
91
|
+
PostedCategory.new(property, category_class, (category[:name].presence || "main").to_s, fields)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Returns [Array<Hash>] the permissible properties of each category of the supplied category class, see .sanitize
|
|
97
|
+
# for `nil_blank`, which only applies to the CLEARABLE_CATEGORY_FIELDS, and `target`, which gets the error of a
|
|
98
|
+
# schema that is not valid JSON.
|
|
99
|
+
#
|
|
100
|
+
# Each value is converted by its field in the category class, such as a Boolean or an Integer rather than the text
|
|
101
|
+
# that the form posts, since a Dirmon entry saves its properties as they are given.
|
|
102
|
+
def self.sanitize_categories(properties, category_class, nil_blank = true, target = nil)
|
|
65
103
|
categories = []
|
|
66
104
|
|
|
67
105
|
properties.each_pair do |_, category|
|
|
@@ -70,16 +108,39 @@ module RocketJobMissionControl
|
|
|
70
108
|
next unless category.key?(key)
|
|
71
109
|
|
|
72
110
|
value = category[key]
|
|
73
|
-
|
|
74
|
-
|
|
111
|
+
# A multi-select posts a blank entry, so that it is posted even with nothing selected.
|
|
112
|
+
value = value.compact_blank if key == :columns && value.is_a?(Array)
|
|
113
|
+
if value.blank?
|
|
114
|
+
hash[key] = nil if nil_blank && CLEARABLE_CATEGORY_FIELDS.include?(key)
|
|
115
|
+
next
|
|
116
|
+
end
|
|
75
117
|
|
|
76
|
-
value
|
|
77
|
-
|
|
118
|
+
value = JSON.parse(value) if key == :format_options
|
|
119
|
+
if key == :schema
|
|
120
|
+
value = sanitize_schema(value, target)
|
|
121
|
+
next if value.nil?
|
|
122
|
+
end
|
|
123
|
+
field = category_class.fields[key.to_s]
|
|
124
|
+
hash[key] = field ? field.mongoize(value) : value
|
|
78
125
|
end
|
|
79
126
|
categories << hash
|
|
80
127
|
end
|
|
81
128
|
|
|
82
129
|
categories
|
|
83
130
|
end
|
|
131
|
+
|
|
132
|
+
# Returns [Hash|String] the schema of a category as its form gives it, see RocketJob::Category::Base#schema: the
|
|
133
|
+
# JSON of a schema, which is parsed, or the path or url of the JSON file that holds it.
|
|
134
|
+
# Returns nil, and adds the error to the supplied target, when the JSON is not valid, so that the schema is not
|
|
135
|
+
# changed.
|
|
136
|
+
def self.sanitize_schema(value, target)
|
|
137
|
+
text = value.strip
|
|
138
|
+
return text unless text.start_with?("{")
|
|
139
|
+
|
|
140
|
+
JSON.parse(text)
|
|
141
|
+
rescue JSON::ParserError => e
|
|
142
|
+
target&.errors&.add(:properties, "Schema is not valid JSON: #{e.message}")
|
|
143
|
+
nil
|
|
144
|
+
end
|
|
84
145
|
end
|
|
85
146
|
end
|