rocketjob_mission_control 7.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/rocket_job_mission_control/application_controller.rb +5 -0
  4. data/app/controllers/rocket_job_mission_control/dirmon_entries_controller.rb +18 -10
  5. data/app/controllers/rocket_job_mission_control/jobs_controller.rb +21 -21
  6. data/app/datatables/rocket_job_mission_control/dirmon_entries_datatable.rb +1 -1
  7. data/app/datatables/rocket_job_mission_control/servers_datatable.rb +1 -1
  8. data/app/helpers/rocket_job_mission_control/application_helper.rb +18 -32
  9. data/app/helpers/rocket_job_mission_control/dirmon_entries_helper.rb +16 -0
  10. data/app/helpers/rocket_job_mission_control/jobs_helper.rb +26 -6
  11. data/app/models/rocket_job_mission_control/dirmon_sanitizer.rb +121 -3
  12. data/app/models/rocket_job_mission_control/job_sanitizer.rb +76 -15
  13. data/app/models/rocket_job_mission_control/record_editor.rb +36 -0
  14. data/app/views/rocket_job_mission_control/application/_schema_proposals.html.erb +74 -0
  15. data/app/views/rocket_job_mission_control/dirmon_entries/_details.html.erb +2 -2
  16. data/app/views/rocket_job_mission_control/dirmon_entries/_form.html.erb +3 -2
  17. data/app/views/rocket_job_mission_control/dirmon_entries/_input_category_fields.html.erb +11 -6
  18. data/app/views/rocket_job_mission_control/dirmon_entries/_output_category_fields.html.erb +8 -2
  19. data/app/views/rocket_job_mission_control/dirmon_entries/_properties.html.erb +3 -2
  20. data/app/views/rocket_job_mission_control/dirmon_entries/show.html.erb +1 -0
  21. data/app/views/rocket_job_mission_control/jobs/_details.html.erb +1 -1
  22. data/app/views/rocket_job_mission_control/jobs/_input_categories.html.erb +10 -4
  23. data/app/views/rocket_job_mission_control/jobs/_input_category_fields.html.erb +17 -5
  24. data/app/views/rocket_job_mission_control/jobs/_output_categories.html.erb +7 -1
  25. data/app/views/rocket_job_mission_control/jobs/_output_category_fields.html.erb +14 -1
  26. data/app/views/rocket_job_mission_control/jobs/edit_slice.html.erb +10 -10
  27. data/app/views/rocket_job_mission_control/jobs/show.html.erb +1 -0
  28. data/lib/rocket_job_mission_control/version.rb +1 -1
  29. metadata +6 -7
  30. data/app/assets/javascripts/rocket_job_mission_control/nested_fields.js +0 -112
  31. data/app/views/rocket_job_mission_control/dirmon_entries/_input_categories.html.erb +0 -59
  32. data/app/views/rocket_job_mission_control/dirmon_entries/_output_categories.html.erb +0 -44
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8cfcccd45da9eb192ca68c39f747d62047a18ed85b454bd9811acd7acabf8a46
4
- data.tar.gz: 97c7242cdf7dc2f7439c8aa5dca7942840fb38e1920dfafb158d655837aa2949
3
+ metadata.gz: 6fb6310a1a33fad99189a35c059a32f6eb77222f27276ca490eb80d61230e1c0
4
+ data.tar.gz: 316fb96d05902f8399717fc0cec7f7987c86906096ad209d40b297d2aad8e90d
5
5
  SHA512:
6
- metadata.gz: cd3f571ed71a27a6a1d0a8d776cf05d970d82dfed3fc4c935e589f7fdd85b224166a86b9e64b8c334a38dbdbb52945ada7528795839e93fffa1984e81f06b99b
7
- data.tar.gz: 68b49776df75c5a989653abcc9a914d5df18a226740c954b4efbecafe0e3fae0351c31ebe650164cf0577dde59af0b3deb151a7ccb8b39a046092587274f1033
6
+ metadata.gz: 427be2abefed3b1eb0845b1fd26dd6258d4dbe06cc43c1cb23f4a5c2ff43d8192b9a6eb394027fd5ea43616543f23f62b2d88f5ff0f36d6c572a0a50bff23b2f
7
+ data.tar.gz: 074d0349f4df4001889ff54809005d5e2fccce394ada49ce4d534147e7c2ca8fb6fdfd3c54500d194fa6672d29629d2eb582d7607996a60ebea7388613403a30
data/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  # Rocket Job Mission Control
2
- [![Gem Version](https://img.shields.io/gem/v/rocketjob_mission_control.svg)](https://rubygems.org/gems/rocketjob_mission_control) [![Build Status](https://github.com/reidmorrison/rocketjob_mission_control/workflows/build/badge.svg)](https://github.com/reidmorrison/rocketjob_mission_control/actions?query=workflow%3Abuild) [![Downloads](https://img.shields.io/gem/dt/rocketjob_mission_control.svg)](https://rubygems.org/gems/rocketjob_mission_control) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
2
+ [![Gem Version](https://img.shields.io/gem/v/rocketjob_mission_control.svg)](https://rubygems.org/gems/rocketjob_mission_control) [![Build Status](https://github.com/reidmorrison/rocketjob_mission_control/workflows/build/badge.svg)](https://github.com/reidmorrison/rocketjob_mission_control/actions?query=workflow%3Abuild) [![Downloads](https://img.shields.io/gem/dt/rocketjob_mission_control.svg)](https://rubygems.org/gems/rocketjob_mission_control) [![Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/reidmorrison/rocketjob_mission_control/badges/coverage.json)](https://github.com/reidmorrison/rocketjob_mission_control/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
3
3
 
4
4
  Rocket Job Mission Control is the web based user interface for [Rocket Job](https://rocketjob.reidmorrison.com/),
5
5
  Ruby's missing batch system.
@@ -2,6 +2,11 @@ module RocketJobMissionControl
2
2
  class ApplicationController < ActionController::Base
3
3
  protect_from_forgery with: :exception
4
4
 
5
+ # The status of a form that is rendered again to show why it could not be saved. Turbo Drive ignores a form
6
+ # response that renders with 200, so the errors would never be shown. Rack 2.2, which Rails 7.2 allows, does not
7
+ # know the name :unprocessable_content, and Rack 3.1 deprecates :unprocessable_entity.
8
+ FORM_ERROR_STATUS = 422
9
+
5
10
  around_action :with_time_zone
6
11
 
7
12
  private
@@ -62,21 +62,22 @@ module RocketJobMissionControl
62
62
  if @dirmon_entry.errors.empty? && @dirmon_entry.save
63
63
  redirect_to(dirmon_entry_path(@dirmon_entry))
64
64
  else
65
- render :new
65
+ render :new, status: FORM_ERROR_STATUS
66
66
  end
67
67
  end
68
68
 
69
69
  def update
70
70
  authorize! :update, @dirmon_entry
71
- sanitized_params = DirmonSanitizer.sanitize(params[:rocket_job_dirmon_entry], @dirmon_entry.job_class, @dirmon_entry)
72
- properties = DirmonSanitizer.diff_properties(sanitized_params[:properties], @dirmon_entry)
71
+ entry_params = params[:rocket_job_dirmon_entry]
72
+ sanitized_params = DirmonSanitizer.sanitize(entry_params, @dirmon_entry.job_class, @dirmon_entry)
73
73
 
74
- sanitized_params[:properties] = properties
74
+ sanitized_params[:properties] =
75
+ DirmonSanitizer.update_properties(entry_params[:properties], sanitized_params[:properties], @dirmon_entry)
75
76
 
76
77
  if @dirmon_entry.errors.empty? && @dirmon_entry.valid? && @dirmon_entry.update(sanitized_params)
77
78
  redirect_to(rocket_job_mission_control.dirmon_entry_path(@dirmon_entry))
78
79
  else
79
- render :edit
80
+ render :edit, status: FORM_ERROR_STATUS
80
81
  end
81
82
  end
82
83
 
@@ -96,11 +97,11 @@ module RocketJobMissionControl
96
97
  # the replicate method clones the existing Dirmon Entity
97
98
  def replicate
98
99
  authorize! :replicate, @dirmon_entry
99
- dirmon_entry_replicate = RocketJob::DirmonEntry.new(@dirmon_entry.dup.attributes.except("id"))
100
+ dirmon_entry_replicate = @dirmon_entry.replicate
100
101
 
101
102
  if (properties = params[:rocket_job_dirmon_entry][:properties])
102
- dirmon_entry_replicate.properties = JobSanitizer.sanitize(properties, dirmon_entry_replicate.job_class, @dirmon_entry,
103
- false)
103
+ sanitized = DirmonSanitizer.sanitize_properties(properties, dirmon_entry_replicate.job_class, @dirmon_entry)
104
+ dirmon_entry_replicate.properties = DirmonSanitizer.update_properties(properties, sanitized, @dirmon_entry)
104
105
  end
105
106
 
106
107
  if dirmon_entry_replicate.update(dirmon_params)
@@ -111,7 +112,7 @@ module RocketJobMissionControl
111
112
  dirmon_entry_replicate.errors.messages.each_pair do |field, message|
112
113
  @dirmon_entry.errors.add(field, message)
113
114
  end
114
- render :copy
115
+ render :copy, status: FORM_ERROR_STATUS
115
116
  end
116
117
  end
117
118
 
@@ -160,6 +161,13 @@ module RocketJobMissionControl
160
161
  params.fetch(:rocket_job_dirmon_entry, {}).permit(:name, :archive_directory, :pattern, :job_class_name, :properties)
161
162
  end
162
163
 
164
+ # The columns of the Dirmon entries list that its search box searches. A pattern can hold credentials, which the
165
+ # list leaves out, see RocketJob::DirmonEntry#pattern_display_name, so it is only searched for a role that can edit
166
+ # an entry, whose form shows the pattern as stored. Otherwise a search could find a password one letter at a time.
167
+ def dirmon_search_columns
168
+ can?(:edit, RocketJob::DirmonEntry) ? %i[job_class_name name pattern] : %i[job_class_name name]
169
+ end
170
+
163
171
  def render_datatable(entries, description)
164
172
  respond_to do |format|
165
173
  format.html do
@@ -169,7 +177,7 @@ module RocketJobMissionControl
169
177
 
170
178
  format.json do
171
179
  query = RocketJobMissionControl::Query.new(entries, name: :asc)
172
- query.search_columns = %i[job_class_name name pattern]
180
+ query.search_columns = dirmon_search_columns
173
181
  query.display_columns = %w[name _type pattern]
174
182
  render(json: DirmonEntriesDatatable.new(view_context, query))
175
183
  end
@@ -83,7 +83,7 @@ module RocketJobMissionControl
83
83
  if @job.errors.empty? && @job.valid? && @job.save
84
84
  redirect_to job_path(@job)
85
85
  else
86
- render :edit
86
+ render :edit, status: FORM_ERROR_STATUS
87
87
  end
88
88
  end
89
89
 
@@ -172,32 +172,32 @@ module RocketJobMissionControl
172
172
  authorize! :update_slice, @job
173
173
 
174
174
  # Params from the edit_slice form
175
- error_type = params[:error_type]
176
- offset = params.fetch(:offset, 0).to_i
177
- updated_records = params["job"]["records"]
175
+ error_type = params[:error_type]
176
+ offset = params.fetch(:offset, 0).to_i
177
+ line_index = params[:line_index].to_i
178
178
 
179
179
  # Find the same slice edit_slice displayed: filtered by error type and
180
180
  # ordered so the offset selects the matching slice.
181
181
  scope = @job.input.failed.where("exception.class_name" => error_type)
182
182
  slice = scope.order(_id: 1).limit(1).skip(offset).first
183
183
 
184
- # Normalize CRLF on the (ASCII-safe) escaped text, then convert the
185
- # \xHH / \\ escapes back into the original bytes. See RecordEscaper.
186
- updated_records = updated_records.map { |record| RecordEscaper.unescape(record.gsub("\r\n", "\n")) }
187
-
188
- # Assings modified slice (from the form) back to slice
189
- slice.records = updated_records
184
+ # Replace only the edited record, converting its text back into a record
185
+ # of the same kind, see RecordEditor. The other records are left as they are.
186
+ records = slice.to_a.dup
187
+ records[line_index] = RecordEditor.from_text(params.dig("job", "record"), records[line_index])
188
+ slice.records = records
190
189
 
191
190
  if slice.save
192
- logger.info("Slice Updated By #{login}, job: #{@job.id}, file_name: #{@job.upload_file_name}")
191
+ logger.info("Slice Updated By #{login}, job: #{@job.id}, file_name: #{helpers.job_upload_file_name(@job)}")
193
192
  flash[:success] = "slice updated"
194
193
  redirect_to view_slice_job_path(@job, error_type: error_type, offset: offset)
195
194
  else
196
195
  flash[:danger] = "Error updating slice."
197
196
  end
198
- rescue EncodingError => e
197
+ rescue EncodingError, JSON::ParserError, BSON::Error => e
199
198
  # Mongo only stores UTF-8, so a record left with invalid bytes cannot be
200
- # saved. Surface it instead of returning a 500.
199
+ # saved, and the JSON of a record that is not text must be valid.
200
+ # Surface it instead of returning a 500.
201
201
  flash[:danger] = "Error updating slice: #{e.message}"
202
202
  redirect_to view_slice_job_path(@job, error_type: params[:error_type], offset: offset)
203
203
  end
@@ -214,15 +214,13 @@ module RocketJobMissionControl
214
214
  scope = @job.input.failed.where("exception.class_name" => error_type)
215
215
  slice = scope.order(_id: 1).limit(1).skip(offset).first
216
216
 
217
- # Finds and deletes line
218
- value = slice.to_a[line_index]
219
- slice.to_a.delete(value)
220
-
221
- # Assings full array back to slice
222
- slice.records = slice.to_a
217
+ # Deletes only the selected record, not every record equal to it.
218
+ records = slice.to_a.dup
219
+ records.delete_at(line_index)
220
+ slice.records = records
223
221
 
224
222
  if slice.save
225
- logger.info("Line Deleted By #{login}, job: #{@job.id}, file_name: #{@job.upload_file_name}")
223
+ logger.info("Line Deleted By #{login}, job: #{@job.id}, file_name: #{helpers.job_upload_file_name(@job)}")
226
224
  flash[:success] = "Record #{slice.first_record_number + line_index} removed from the slice."
227
225
  redirect_to view_slice_job_path(@job, error_type: error_type, offset: offset), status: :see_other
228
226
  else
@@ -301,7 +299,8 @@ module RocketJobMissionControl
301
299
  :name,
302
300
  :format,
303
301
  :format_options,
304
- :mode,
302
+ :schema,
303
+ :validate_rows,
305
304
  :skip_unknown,
306
305
  :slice_size,
307
306
  {columns: []}
@@ -311,6 +310,7 @@ module RocketJobMissionControl
311
310
  :name,
312
311
  :format,
313
312
  :format_options,
313
+ :schema,
314
314
  {columns: []}
315
315
  ]
316
316
  )
@@ -8,7 +8,7 @@ module RocketJobMissionControl
8
8
  {
9
9
  "0" => name_with_link(dirmon),
10
10
  "1" => h(dirmon.job_class_name),
11
- "2" => h(dirmon.pattern.try(:truncate, 80)),
11
+ "2" => h(dirmon.pattern_display_name.try(:truncate, 80)),
12
12
  "DT_RowClass" => "rjmc-card callout callout-#{dirmon.state}"
13
13
  }
14
14
  end
@@ -62,7 +62,7 @@ module RocketJobMissionControl
62
62
 
63
63
  if server.stopping? && view.can?(:destroy, server)
64
64
  actions += "Server is stopping..."
65
- confirmation = ""
65
+ confirmation = +""
66
66
  unless server.zombie?
67
67
  confirmation << "Warning!\n\nDestroying this server will hard kill its active workers/jobs.\nKilled jobs will be requeued for processing on another worker.\n\n"
68
68
  end
@@ -83,6 +83,24 @@ module RocketJobMissionControl
83
83
  values
84
84
  end
85
85
 
86
+ # Returns the text area for the schema of a category, see RocketJob::Category::Base#schema: the JSON of a schema,
87
+ # or the path or url of the JSON file that holds it. Blank is no schema.
88
+ def schema_text_area(f, value)
89
+ text = value.is_a?(Hash) ? JSON.pretty_generate(value) : value
90
+ f.text_area(:schema, value: text, rows: 4, class: "form-control font-monospace",
91
+ placeholder: '{"fields": [{"name": "name"}, {"name": "age", "type": "integer"}]}')
92
+ end
93
+
94
+ # Returns the select for the validate_rows of an input category, see RocketJob::Category::Input#validate_rows:
95
+ # blank, which clears it on a job and leaves it to the job class on a Dirmon entry, true or false. A category without
96
+ # one whose upload validates its lines anyway, such as one with a schema in the deprecated :hash mode, shows true,
97
+ # as Rocket Job 8 reads it, so that the form shows what the category does.
98
+ def validate_rows_select(f, category)
99
+ value = category.validate_rows
100
+ value = true if value.nil? && category.upload_validates_rows?
101
+ f.select(:validate_rows, [true, false], {include_blank: true, selected: value}, {class: "tom-select form-select"})
102
+ end
103
+
86
104
  # Returns the editable field as html for use in editing dynamic fields from a Job class.
87
105
  def editable_field_html(klass, field_name, value, f)
88
106
  # When editing a job the values are of the correct type.
@@ -123,37 +141,5 @@ placeholder: '{"key1":"value1", "key2":"value2", "key3":"value3"}')
123
141
  f.text_field(field_name, value: value, class: "form-control", placeholder: placeholder)
124
142
  end
125
143
  end
126
-
127
- # This method creates a link with `data-id` `data-fields` attributes. These attributes are used to create new instances of the nested fields through Javascript.
128
- def link_to_add_fields(name, f, association, option)
129
- # Takes an object (@job) and creates a new instance of its associated model (:properties)
130
- new_object = f.object.send(association).klass.new
131
-
132
- # Saves the unique ID of the object into a variable.
133
- # This is needed to ensure the key of the associated array is unique. This is makes parsing the content in the `data-fields` attribute easier through Javascript.
134
- # We could use another method to achive this.
135
- id = new_object.object_id
136
-
137
- # https://api.rubyonrails.org/ fields_for(record_name, record_object = nil, fields_options = {}, &block)
138
- # record_name = :addresses
139
- # record_object = new_object
140
- # fields_options = { child_index: id }
141
- # child_index` is used to ensure the key of the associated array is unique, and that it matched the value in the `data-id` attribute.
142
- # `person[addresses_attributes][child_index_value][_destroy]`
143
- fields = f.fields_for(association, new_object, child_index: id) do |builder|
144
- # `association.to_s.singularize + "_fields"` ends up evaluating to `address_fields`
145
- # The render function will then look for `views/people/_address_fields.html.erb`
146
- # The render function also needs to be passed the value of 'builder', because `views/dirmon_entries/_input_categories.html.erb` needs this to render the form tags.
147
- render("#{association.to_s.singularize}_fields", f: builder)
148
- end
149
-
150
- # This renders a simple link, but passes information into `data` attributes.
151
- # This info can be named anything we want, but in this case we chose `data-id:` and `data-fields:`.
152
- # The `id:` is from `new_object.object_id`.
153
- # The `fields:` are rendered from the `fields` blocks.
154
- # We use `gsub("\n", "")` to remove anywhite space from the rendered partial.
155
- # The `id:` value needs to match the value used in `child_index: id`.
156
- link_to(name, "#", class: "add_fields btn btn-#{option}", data: {id: id, fields: fields.delete("\n")})
157
- end
158
144
  end
159
145
  end
@@ -11,6 +11,22 @@ module RocketJobMissionControl
11
11
  nil
12
12
  end
13
13
 
14
+ # Returns [RocketJob::Job] the job that the entry's properties build, whose values its form shows, or nil without a
15
+ # job class. Returns nil too when Rocket Job cannot build one from them, such as for a category that the job class
16
+ # does not define or an option that it no longer has, and adds why to the entry's errors, so that its form still
17
+ # shows its other attributes rather than failing. The error is logged with its backtrace too, since it can come from
18
+ # the job class itself, such as a mistake in a default of one of its fields.
19
+ def dirmon_entry_job(dirmon_entry)
20
+ dirmon_entry.job_class&.from_properties(dirmon_entry.properties)
21
+ rescue ArgumentError, NoMethodError, Mongoid::Errors::MongoidError => e
22
+ job_class_name = dirmon_entry.job_class_name
23
+ logger.warn("Dirmon entry #{dirmon_entry.id}: its properties cannot build a #{job_class_name}: #{e.class}: " \
24
+ "#{e.message}\n#{Array(e.backtrace).join("\n")}")
25
+ reason = e.respond_to?(:problem) ? e.problem : e.message
26
+ dirmon_entry.errors.add(:properties, "cannot build a #{job_class_name}, so they cannot be edited here: #{reason}")
27
+ nil
28
+ end
29
+
14
30
  def rocket_job_mission_control
15
31
  @@rocket_job_mission_control_engine_url_helpers ||= RocketJobMissionControl::Engine.routes.url_helpers
16
32
  end
@@ -107,8 +107,27 @@ module RocketJobMissionControl
107
107
  (job.record_count.to_f / job.input_category.slice_size).ceil
108
108
  end
109
109
 
110
+ # The job's upload file name, without any credentials, such as the password of an SFTP url that Dirmon
111
+ # archived the file to, see RocketJob.path_display_name.
112
+ def job_upload_file_name(job)
113
+ RocketJob.path_display_name(job.upload_file_name)
114
+ end
115
+
116
+ # The category's file name, without any credentials, such as the password of an SFTP url that the output is
117
+ # downloaded to. The category decides how its file name is shown, see RocketJob::Category::Base.display_properties.
118
+ def job_category_file_name(category)
119
+ category.class.display_properties("file_name" => category.file_name)["file_name"]
120
+ end
121
+
122
+ # The category's schema: its fields as a JSON tree, or the path or url of the JSON file that holds it, without any
123
+ # credentials. The category decides how its schema is shown, see RocketJob::Category::Base.display_properties.
124
+ def job_category_schema(category)
125
+ schema = category.class.display_properties("schema" => category.schema)["schema"]
126
+ schema.is_a?(Hash) ? render_json_tree(schema) : schema
127
+ end
128
+
110
129
  def job_custom_fields(job)
111
- attrs = job.attributes.dup
130
+ attrs = job.display_attributes
112
131
  DISPLAYED_FIELDS.each { |key| attrs.delete(key) }
113
132
  # Convert time zones for any custom time fields
114
133
  attrs.keys { |key| attrs[key] = attrs[key].in_time_zone(Time.zone) if attrs[key].is_a?(Time) }
@@ -183,19 +202,20 @@ module RocketJobMissionControl
183
202
  backtrace_strip_paths.any? { |path| line.to_s.start_with?(path) }
184
203
  end
185
204
 
186
- # Reversible, ASCII-safe form of a failed record for an edit textarea.
187
- # See RecordEscaper; unescape it with RecordEscaper.unescape on save.
205
+ # Reversible form of a failed record for an edit textarea, which keeps its type.
206
+ # See RecordEditor; convert it back with RecordEditor.from_text on save.
188
207
  def escape_record(value)
189
- RecordEscaper.escape(value)
208
+ RecordEditor.to_text(value)
190
209
  end
191
210
 
192
211
  # Read-only HTML for a failed record, with each unprintable/invalid byte
193
212
  # rendered as a highlighted `\xHH` token. Each highlight carries a hover
194
213
  # tooltip naming the byte, so an operator can see (in context) exactly which
195
- # byte broke the record and what it is.
214
+ # byte broke the record and what it is. A record that is not a String, such as
215
+ # a Hash, is shown as Extended JSON, see RecordEditor.
196
216
  def highlight_record(value)
197
217
  safe_join(
198
- RecordEscaper.segments(value).map do |type, text|
218
+ RecordEscaper.segments(RecordEditor.to_display_text(value)).map do |type, text|
199
219
  next text if type == :text
200
220
 
201
221
  content_tag(:span, text, class: "record-escape", title: record_escape_title(text))
@@ -13,17 +13,52 @@ module RocketJobMissionControl
13
13
  end
14
14
 
15
15
  if params.key?(:properties)
16
- properties = JobSanitizer.sanitize(params[:properties], job_class, target, false)
16
+ properties = sanitize_properties(params[:properties], job_class, target)
17
17
  permissible_params[:properties] = properties if properties.present?
18
18
  end
19
19
 
20
20
  permissible_params
21
21
  end
22
22
 
23
+ # Returns [Hash] the supplied properties that a form posted, sanitized for a Dirmon entry, see JobSanitizer.sanitize,
24
+ # without those posted blank, which .update_properties removes from the entry.
25
+ def self.sanitize_properties(posted, job_class, target)
26
+ JobSanitizer.sanitize(posted, job_class, target, false)
27
+ end
28
+
29
+ # Returns [Hash] the properties of the supplied Dirmon entry with the changes that its form posted, see
30
+ # .diff_properties: each property that the form posted is set when it differs from the job class's default, and
31
+ # removed otherwise, so that the job class decides it. Every other property is kept as it is saved, such as an
32
+ # input category's `allowed_columns` or `file_name`, since the form does not show it.
33
+ #
34
+ # Parameters
35
+ # posted [Hash]
36
+ # The properties that the form posted, see JobSanitizer.posted_fields and .posted_categories.
37
+ #
38
+ # sanitized [Hash]
39
+ # The same properties after .sanitize_properties.
40
+ #
41
+ # dirmon_entry [RocketJob::DirmonEntry]
42
+ # The entry whose properties the form shows, such as the one being edited or copied.
43
+ def self.update_properties(posted, sanitized, dirmon_entry)
44
+ properties = dirmon_entry.properties.to_h.deep_stringify_keys
45
+ return properties if posted.blank?
46
+
47
+ changes = diff_properties(sanitized, dirmon_entry).deep_stringify_keys
48
+ JobSanitizer.posted_fields(posted, dirmon_entry.job_class).each do |name|
49
+ merge_property(properties, changes, name.to_s)
50
+ end
51
+ JobSanitizer.posted_categories(posted).each do |category|
52
+ merge_category(properties, category, changes, dirmon_entry.job_class)
53
+ end
54
+ properties
55
+ end
56
+
23
57
  # Returns [Hash] the difference between the supplied params and those already set in the job itself
24
58
  def self.diff_properties(sanitized_properties, dirmon_entry)
25
59
  default_job = dirmon_entry.job_class.new
26
- updated_job = dirmon_entry.job_class.from_properties(sanitized_properties)
60
+ # Without properties to change, such as when only the pattern changed, the job has the defaults.
61
+ updated_job = dirmon_entry.job_class.from_properties(sanitized_properties || {})
27
62
  properties = {}
28
63
  sanitized_properties&.each_pair do |name, value|
29
64
  if name == :input_categories
@@ -45,7 +80,7 @@ module RocketJobMissionControl
45
80
  end
46
81
  properties[:output_categories] = categories unless categories.empty?
47
82
  elsif default_job.public_send(name) != updated_job.public_send(name)
48
- properties[name] = value.is_a?(String) ? value.gsub("\r\n", "\n") : value
83
+ properties[name] = value
49
84
  end
50
85
  end
51
86
  properties
@@ -66,5 +101,88 @@ module RocketJobMissionControl
66
101
  diff[:name] = name unless diff.empty?
67
102
  diff
68
103
  end
104
+
105
+ # Merges the changes that the form posted for the supplied category, see JobSanitizer.posted_categories, into the
106
+ # supplied properties, see .update_properties. A saved category is found by its name, or is main without one, as a
107
+ # job reads it, and is removed when it is left with only its name, since a job then has the job class's category.
108
+ #
109
+ # A category with a schema cannot have the options that the schema replaces, such as an input category's
110
+ # `allowed_columns`, see RocketJob::Category::Input.deprecated_column_options. Those that the form does not show are
111
+ # removed, since they could not be changed otherwise, so that giving a category a schema replaces them.
112
+ #
113
+ # A validate_rows of true that the form shows for a category that its job class already validates is removed, see
114
+ # .remove_inherited_validate_rows.
115
+ def self.merge_category(properties, posted, changes, job_class)
116
+ key = posted.property.to_s
117
+ categories = Array(properties[key]).dup
118
+ change = Array(changes[key]).find { |category| category["name"].to_s == posted.name } || {}
119
+ index = categories.index { |category| category.is_a?(Hash) && (category["name"] || "main").to_s == posted.name }
120
+ category = index ? categories[index].dup : {"name" => posted.name}
121
+
122
+ fields = posted.fields.map(&:to_s)
123
+ fields.each { |field| merge_property(category, change, field) }
124
+ remove_options_replaced_by_schema(category, posted, job_class)
125
+ remove_inherited_validate_rows(category, posted, job_class)
126
+
127
+ if (category.keys - ["name"]).empty?
128
+ categories.delete_at(index) if index
129
+ elsif index
130
+ categories[index] = category
131
+ else
132
+ categories << category
133
+ end
134
+ categories.empty? ? properties.delete(key) : properties[key] = categories
135
+ end
136
+
137
+ # Removes the options that the schema of the supplied category replaces, see .merge_category: its own schema, or else
138
+ # the one that its job class gives it. Those that the form posted stay, such as an input category's `skip_unknown`,
139
+ # since it can change them.
140
+ def self.remove_options_replaced_by_schema(category, posted, job_class)
141
+ schema = category["schema"].presence || job_class_category(job_class, posted)&.schema
142
+ return if schema.blank?
143
+
144
+ replaced = posted.category_class.deprecated_column_options(category.merge("schema" => schema))
145
+ category.except!(*(replaced - posted.fields.map(&:to_s)))
146
+ end
147
+
148
+ # Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category of a new job of the supplied class
149
+ # that the supplied posted category changes, see JobSanitizer.posted_categories, or nil when it has none.
150
+ def self.job_class_category(job_class, posted)
151
+ job_class.new.public_send(posted.property).find { |category| category.name.to_s == posted.name }
152
+ end
153
+
154
+ # Removes the validate_rows of true that the form shows for an input category that validates its lines without it,
155
+ # see ApplicationHelper#validate_rows_select, when its job class with the fields that the form shows already
156
+ # validates them, such as a job class that defines the category in the deprecated :hash mode, so that the entry keeps
157
+ # following its job class. It is kept when the entry's own properties that the form does not show are what validate
158
+ # them, such as a saved :hash mode, or would stop it, such as a mode of their own.
159
+ def self.remove_inherited_validate_rows(category, posted, job_class)
160
+ return unless category["validate_rows"] == true && posted.fields.include?(:validate_rows)
161
+
162
+ without = category.except("validate_rows")
163
+ shown = without.slice("name", *posted.fields.map(&:to_s))
164
+ return unless validates_rows?(job_class, posted, shown) && validates_rows?(job_class, posted, without)
165
+
166
+ category.delete("validate_rows")
167
+ end
168
+
169
+ # Returns [true|false] whether a job of the supplied class, built with the supplied properties of the posted input
170
+ # category, validates the lines of each file that it uploads, see RocketJob::Category::Input#upload_validates_rows?.
171
+ def self.validates_rows?(job_class, posted, category)
172
+ job = job_class.from_properties(posted.property.to_s => [category])
173
+ job.input_category(posted.name.to_sym).upload_validates_rows?
174
+ end
175
+
176
+ # Sets the supplied property to its change, or removes it when the form did not change it from the default.
177
+ def self.merge_property(properties, changes, key)
178
+ if changes.key?(key)
179
+ properties[key] = changes[key]
180
+ else
181
+ properties.delete(key)
182
+ end
183
+ end
184
+
185
+ private_class_method :merge_category, :remove_options_replaced_by_schema, :job_class_category,
186
+ :remove_inherited_validate_rows, :validates_rows?, :merge_property
69
187
  end
70
188
  end
@@ -1,6 +1,21 @@
1
1
  module RocketJobMissionControl
2
2
  module JobSanitizer
3
- CATEGORIES_FIELDS = %i[id name format format_options mode skip_unknown slice_size columns].freeze
3
+ CATEGORIES_FIELDS = %i[id name format format_options schema validate_rows skip_unknown slice_size columns].freeze
4
+
5
+ # The category fields whose blank value means their default, nil, such as no `schema`, so that blanking them
6
+ # clears a value that was set.
7
+ CLEARABLE_CATEGORY_FIELDS = %i[schema validate_rows].freeze
8
+
9
+ # The param that the job and Dirmon entry forms post each kind of category in, by the property that holds those
10
+ # categories, with their class.
11
+ CATEGORY_PARAMS = {
12
+ input_categories: [:input_categories_attributes, RocketJob::Category::Input],
13
+ output_categories: [:output_categories_attributes, RocketJob::Category::Output]
14
+ }.freeze
15
+
16
+ # A category that a form posted, see .posted_categories: the property that holds it, such as :input_categories,
17
+ # its class, its name, and the names of its fields that the form posted.
18
+ PostedCategory = Struct.new(:property, :category_class, :name, :fields)
4
19
 
5
20
  # Returns [Hash] the permissible params for the specified job class, after sanitizing.
6
21
  # Parameters
@@ -27,8 +42,8 @@ module RocketJobMissionControl
27
42
  next unless field&.type
28
43
 
29
44
  case field.type.name
30
- when "String"
31
- value.gsub("\r\n", "\n")
45
+ when "String", "Symbol", "Mongoid::StringifiedSymbol"
46
+ value = value.gsub("\r\n", "\n")
32
47
  when "Hash"
33
48
  begin
34
49
  value = value.blank? ? nil : JSON.parse(value)
@@ -48,20 +63,43 @@ module RocketJobMissionControl
48
63
  end
49
64
  end
50
65
 
51
- if properties.key?(:input_categories_attributes)
52
- categories = sanitize_categories(properties[:input_categories_attributes])
53
- permissible_params[:input_categories] = categories unless categories == [{}]
54
- end
66
+ CATEGORY_PARAMS.each_pair do |property, (param, category_class)|
67
+ next unless properties.key?(param)
55
68
 
56
- if properties.key?(:output_categories_attributes)
57
- categories = sanitize_categories(properties[:output_categories_attributes])
58
- permissible_params[:output_categories] = categories unless categories == [{}]
69
+ categories = sanitize_categories(properties[param], category_class, nil_blank, target)
70
+ permissible_params[property] = categories unless categories == [{}]
59
71
  end
60
72
 
61
73
  permissible_params
62
74
  end
63
75
 
64
- def self.sanitize_categories(properties)
76
+ # Returns [Array<Symbol>] the user editable fields of the supplied job class that the supplied properties of a form
77
+ # post, see .sanitize, also those posted blank, which .sanitize can leave out.
78
+ def self.posted_fields(properties, job_class)
79
+ job_class.user_editable_fields.select { |field_name| properties.key?(field_name) }
80
+ end
81
+
82
+ # Returns [Array<PostedCategory>] each category that the supplied properties of a form post, see .sanitize, with
83
+ # the names of its fields that the form posted, see CATEGORIES_FIELDS, also those posted blank, which .sanitize can
84
+ # leave out.
85
+ def self.posted_categories(properties)
86
+ CATEGORY_PARAMS.flat_map do |property, (param, category_class)|
87
+ next [] unless properties.key?(param)
88
+
89
+ properties[param].each_pair.map do |_, category|
90
+ fields = (CATEGORIES_FIELDS - %i[id name]).select { |key| category.key?(key) }
91
+ PostedCategory.new(property, category_class, (category[:name].presence || "main").to_s, fields)
92
+ end
93
+ end
94
+ end
95
+
96
+ # Returns [Array<Hash>] the permissible properties of each category of the supplied category class, see .sanitize
97
+ # for `nil_blank`, which only applies to the CLEARABLE_CATEGORY_FIELDS, and `target`, which gets the error of a
98
+ # schema that is not valid JSON.
99
+ #
100
+ # Each value is converted by its field in the category class, such as a Boolean or an Integer rather than the text
101
+ # that the form posts, since a Dirmon entry saves its properties as they are given.
102
+ def self.sanitize_categories(properties, category_class, nil_blank = true, target = nil)
65
103
  categories = []
66
104
 
67
105
  properties.each_pair do |_, category|
@@ -70,16 +108,39 @@ module RocketJobMissionControl
70
108
  next unless category.key?(key)
71
109
 
72
110
  value = category[key]
73
- next if value.blank?
74
- next if (key == :columns) && value == [""]
111
+ # A multi-select posts a blank entry, so that it is posted even with nothing selected.
112
+ value = value.compact_blank if key == :columns && value.is_a?(Array)
113
+ if value.blank?
114
+ hash[key] = nil if nil_blank && CLEARABLE_CATEGORY_FIELDS.include?(key)
115
+ next
116
+ end
75
117
 
76
- value = JSON.parse(value) if key == :format_options
77
- hash[key] = value
118
+ value = JSON.parse(value) if key == :format_options
119
+ if key == :schema
120
+ value = sanitize_schema(value, target)
121
+ next if value.nil?
122
+ end
123
+ field = category_class.fields[key.to_s]
124
+ hash[key] = field ? field.mongoize(value) : value
78
125
  end
79
126
  categories << hash
80
127
  end
81
128
 
82
129
  categories
83
130
  end
131
+
132
+ # Returns [Hash|String] the schema of a category as its form gives it, see RocketJob::Category::Base#schema: the
133
+ # JSON of a schema, which is parsed, or the path or url of the JSON file that holds it.
134
+ # Returns nil, and adds the error to the supplied target, when the JSON is not valid, so that the schema is not
135
+ # changed.
136
+ def self.sanitize_schema(value, target)
137
+ text = value.strip
138
+ return text unless text.start_with?("{")
139
+
140
+ JSON.parse(text)
141
+ rescue JSON::ParserError => e
142
+ target&.errors&.add(:properties, "Schema is not valid JSON: #{e.message}")
143
+ nil
144
+ end
84
145
  end
85
146
  end