rocketjob 5.4.0.beta2 → 6.0.0.rc3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +149 -5
- data/bin/rocketjob_batch_perf +1 -1
- data/bin/rocketjob_perf +1 -1
- data/lib/rocket_job/batch.rb +3 -1
- data/lib/rocket_job/batch/categories.rb +341 -0
- data/lib/rocket_job/batch/io.rb +128 -60
- data/lib/rocket_job/batch/model.rb +20 -68
- data/lib/rocket_job/batch/performance.rb +19 -7
- data/lib/rocket_job/batch/statistics.rb +34 -12
- data/lib/rocket_job/batch/tabular.rb +2 -0
- data/lib/rocket_job/batch/tabular/input.rb +8 -6
- data/lib/rocket_job/batch/tabular/output.rb +4 -2
- data/lib/rocket_job/batch/throttle_running_workers.rb +8 -17
- data/lib/rocket_job/batch/worker.rb +27 -24
- data/lib/rocket_job/category/base.rb +78 -0
- data/lib/rocket_job/category/input.rb +110 -0
- data/lib/rocket_job/category/output.rb +25 -0
- data/lib/rocket_job/cli.rb +25 -17
- data/lib/rocket_job/dirmon_entry.rb +22 -12
- data/lib/rocket_job/event.rb +1 -1
- data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
- data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
- data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
- data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
- data/lib/rocket_job/jobs/conversion_job.rb +39 -0
- data/lib/rocket_job/jobs/dirmon_job.rb +2 -2
- data/lib/rocket_job/jobs/housekeeping_job.rb +7 -7
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +17 -6
- data/lib/rocket_job/jobs/on_demand_job.rb +1 -2
- data/lib/rocket_job/jobs/performance_job.rb +3 -1
- data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -96
- data/lib/rocket_job/jobs/upload_file_job.rb +44 -8
- data/lib/rocket_job/lookup_collection.rb +69 -0
- data/lib/rocket_job/plugins/job/model.rb +25 -50
- data/lib/rocket_job/plugins/job/throttle.rb +2 -2
- data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +12 -4
- data/lib/rocket_job/plugins/job/worker.rb +2 -7
- data/lib/rocket_job/plugins/restart.rb +12 -5
- data/lib/rocket_job/plugins/state_machine.rb +2 -1
- data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +38 -0
- data/lib/rocket_job/ractor_worker.rb +42 -0
- data/lib/rocket_job/server/model.rb +1 -1
- data/lib/rocket_job/sliced.rb +15 -70
- data/lib/rocket_job/sliced/bzip2_output_slice.rb +1 -1
- data/lib/rocket_job/sliced/input.rb +1 -1
- data/lib/rocket_job/sliced/slice.rb +5 -13
- data/lib/rocket_job/sliced/slices.rb +14 -2
- data/lib/rocket_job/sliced/writer/output.rb +33 -45
- data/lib/rocket_job/subscribers/server.rb +1 -1
- data/lib/rocket_job/thread_worker.rb +46 -0
- data/lib/rocket_job/throttle_definitions.rb +7 -1
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +21 -55
- data/lib/rocket_job/worker_pool.rb +5 -7
- data/lib/rocketjob.rb +53 -43
- metadata +36 -26
- data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
- data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6a04a33b0cd03bdf0a7cb948fc87dd6c7d7bb3b392e566a8c15df50b73e27459
|
4
|
+
data.tar.gz: fc62e740a0a92bae8daf1f4ffbe199af1debcb84f8859aed10ea5954dc44c7b6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 74cac01d253cf21a856e1ca4a5cf63d5e90320303bdf310cf90325c9cca242c4ed1b7a0a1c43ca00764f2f40d29822df6e6bee499c1bff56c9ddaa2401bc3862
|
7
|
+
data.tar.gz: 1bbc47c7d869ef28fd578a7b2575f62957aa2f83f9fc927af1d6fba7866270b15fd21cef30007b78d58847137357c75ccae2d03545560d2ffe0d674fe34c1d0e
|
data/README.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# Rocket Job
|
2
|
-
[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![
|
2
|
+
[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Support](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
|
3
3
|
|
4
4
|
Ruby's missing batch system
|
5
5
|
|
@@ -17,11 +17,153 @@ Checkout https://rocketjob.io/
|
|
17
17
|
* Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
|
18
18
|
* [Report bugs](https://github.com/rocketjob/rocketjob/issues)
|
19
19
|
|
20
|
-
## Rocket Job
|
20
|
+
## Rocket Job v6
|
21
21
|
|
22
|
-
|
22
|
+
- Support for Ruby v3 and Rails 6.
|
23
|
+
- Major enhancements in Batch job support:
|
24
|
+
- Direct built-in Tabular support for all input and output categories.
|
25
|
+
- Multiple output file support, each with its own settings for:
|
26
|
+
- Compression
|
27
|
+
- GZip, Zip, BZip2 (Chunked for much faster loading into Apache Spark).
|
28
|
+
- Encryption
|
29
|
+
- PGP, Symmetric Encryption.
|
30
|
+
- File format
|
31
|
+
- CSV, PSV, JSON, Fixed Format, xlsx.
|
32
|
+
- Significant error handling improvements, especially around throttle failures
|
33
|
+
that used to result in "hanging" jobs.
|
34
|
+
- Support AWS DocumentDB in addition to MongoDB as the data store.
|
35
|
+
- Removed use of Symbols to meet Symbol deprecation in MongoDB and Mongoid.
|
23
36
|
|
24
|
-
|
37
|
+
### Upgrading to Rocket Job v6
|
38
|
+
|
39
|
+
The following plugins have been deprecated and are no longer loaded by default.
|
40
|
+
- `RocketJob::Batch::Tabular::Input`
|
41
|
+
- `RocketJob::Batch::Tabular::Output`
|
42
|
+
|
43
|
+
If your code relies on these plugins and you still want to upgrade to Rocket Job v6,
|
44
|
+
add the following require statement to any jobs that still use them:
|
45
|
+
|
46
|
+
~~~ruby
|
47
|
+
require "rocket_job/batch/tabular"
|
48
|
+
~~~
|
49
|
+
|
50
|
+
It is important to migrate away from these plugins, since they will be removed in a future release.
|
51
|
+
|
52
|
+
#### Upgrading Batch Jobs to Rocket Job v6
|
53
|
+
|
54
|
+
Rocket Job v6 replaces the array of symbol type for `input_categories` and `output_categories`
|
55
|
+
with an array of `RocketJob::Category::Input` and `RocketJob::Category::Output`.
|
56
|
+
|
57
|
+
Jobs that added or modified the input or output categories need to be upgraded. For example:
|
58
|
+
~~~ruby
|
59
|
+
class MyJob < RocketJob::Job
|
60
|
+
include RocketJob::Batch
|
61
|
+
|
62
|
+
self.output_categories = [:main, :errors, :ignored]
|
63
|
+
end
|
64
|
+
~~~
|
65
|
+
|
66
|
+
Needs to be changed to:
|
67
|
+
~~~ruby
|
68
|
+
class MyJob < RocketJob::Job
|
69
|
+
include RocketJob::Batch
|
70
|
+
|
71
|
+
output_category name: :main
|
72
|
+
output_category name: :errors
|
73
|
+
output_category name: :ignored
|
74
|
+
end
|
75
|
+
~~~
|
76
|
+
|
77
|
+
##### slice_size, encrypt, compress
|
78
|
+
|
79
|
+
These fields have been removed from the job itself:
|
80
|
+
~~~ruby
|
81
|
+
class MyJob < RocketJob::Job
|
82
|
+
include RocketJob::Batch
|
83
|
+
|
84
|
+
self.slice_sice = 1_000
|
85
|
+
self.encrypt = true
|
86
|
+
self.compress = true
|
87
|
+
end
|
88
|
+
~~~
|
89
|
+
|
90
|
+
They are now specified on the `input_category` as follows:
|
91
|
+
- `slice_size` just moves under `input_category`.
|
92
|
+
- `encrypt` becomes an option to `serializer`.
|
93
|
+
- `compress` is now the default for all batch jobs so is not needed.
|
94
|
+
|
95
|
+
If the serializer is set to `encrypt` then it is automatically compressed.
|
96
|
+
|
97
|
+
~~~ruby
|
98
|
+
class MyJob < RocketJob::Job
|
99
|
+
include RocketJob::Batch
|
100
|
+
|
101
|
+
input_category slice_sice: 1_000, serializer: :encrypt
|
102
|
+
end
|
103
|
+
~~~
|
104
|
+
|
105
|
+
##### collect_output, collect_nil_output
|
106
|
+
|
107
|
+
The following fields have been moved from the job itself:
|
108
|
+
~~~ruby
|
109
|
+
class MyJob < RocketJob::Job
|
110
|
+
include RocketJob::Batch
|
111
|
+
|
112
|
+
self.collect_output = true
|
113
|
+
self.collect_nil_output = true
|
114
|
+
end
|
115
|
+
~~~
|
116
|
+
|
117
|
+
Into the corresponding `output_category`:
|
118
|
+
- `collect_output` no longer has any meaning. Output is collected anytime an `output_category` is defined.
|
119
|
+
- `collect_nil_output` is now the option `nils` on the `output_category.
|
120
|
+
It defaults to `false` so that by default any `nil` output from the `perform` method is not collected.
|
121
|
+
~~~ruby
|
122
|
+
class MyJob < RocketJob::Job
|
123
|
+
include RocketJob::Batch
|
124
|
+
|
125
|
+
output_category nils: true
|
126
|
+
end
|
127
|
+
~~~
|
128
|
+
|
129
|
+
##### name
|
130
|
+
|
131
|
+
For both `input_category` and `output_category`, when the `name` argument is not supplied
|
132
|
+
it defaults to `:main`.
|
133
|
+
|
134
|
+
For Example:
|
135
|
+
~~~ruby
|
136
|
+
class MyJob < RocketJob::Job
|
137
|
+
include RocketJob::Batch
|
138
|
+
|
139
|
+
input_category name: :main, serializer: :encrypt
|
140
|
+
output_category name: :main
|
141
|
+
end
|
142
|
+
~~~
|
143
|
+
|
144
|
+
Is the same as:
|
145
|
+
~~~ruby
|
146
|
+
class MyJob < RocketJob::Job
|
147
|
+
include RocketJob::Batch
|
148
|
+
|
149
|
+
input_category serializer: :encrypt
|
150
|
+
output_category
|
151
|
+
end
|
152
|
+
~~~
|
153
|
+
|
154
|
+
##### Existing and inflight jobs
|
155
|
+
|
156
|
+
When migrating to Rocket Job 6, it is recommended to load every job and then save it back again as part of the
|
157
|
+
deployment. When the job loads it will automatically convert itself from the old schema to the new v6 schema.
|
158
|
+
|
159
|
+
In flight jobs should not be affected, other than it is important to shutdown all running batch
|
160
|
+
servers _before_ running any new instances.
|
161
|
+
|
162
|
+
## Rocket Job v4
|
163
|
+
|
164
|
+
Rocket Job Pro is now fully open source and included in Rocket Job under the Apache License.
|
165
|
+
|
166
|
+
The `RocketJob::Batch` plugin now adds batch processing capabilities to break up a single task into many
|
25
167
|
concurrent workers processing slices of the entire job at the same time.
|
26
168
|
|
27
169
|
|
@@ -33,7 +175,9 @@ class MyJob < RocketJob::Job
|
|
33
175
|
|
34
176
|
self.description = "Reverse names"
|
35
177
|
self.destroy_on_complete = false
|
36
|
-
|
178
|
+
|
179
|
+
# Collect the output for this job in the default output category: `:main`
|
180
|
+
output_category
|
37
181
|
|
38
182
|
# Method to call by all available workers at the same time.
|
39
183
|
# Reverse the characters for each line:
|
data/bin/rocketjob_batch_perf
CHANGED
data/bin/rocketjob_perf
CHANGED
data/lib/rocket_job/batch.rb
CHANGED
@@ -7,6 +7,8 @@ require "rocket_job/batch/state_machine"
|
|
7
7
|
require "rocket_job/batch/throttle"
|
8
8
|
require "rocket_job/batch/throttle_running_workers"
|
9
9
|
require "rocket_job/batch/worker"
|
10
|
+
# Ensure after_perform is run first and #upload override is after IO#upload is defined.
|
11
|
+
require "rocket_job/batch/categories"
|
10
12
|
|
11
13
|
module RocketJob
|
12
14
|
module Batch
|
@@ -17,6 +19,7 @@ module RocketJob
|
|
17
19
|
include Callbacks
|
18
20
|
include Logger
|
19
21
|
include Worker
|
22
|
+
include Categories
|
20
23
|
include Throttle
|
21
24
|
include ThrottleRunningWorkers
|
22
25
|
include IO
|
@@ -27,6 +30,5 @@ module RocketJob
|
|
27
30
|
autoload :ThrottleWindows, "rocket_job/batch/throttle_windows"
|
28
31
|
autoload :Result, "rocket_job/batch/result"
|
29
32
|
autoload :Results, "rocket_job/batch/results"
|
30
|
-
autoload :Tabular, "rocket_job/batch/tabular"
|
31
33
|
end
|
32
34
|
end
|
@@ -0,0 +1,341 @@
|
|
1
|
+
require "active_support/concern"
|
2
|
+
|
3
|
+
module RocketJob
|
4
|
+
module Batch
|
5
|
+
module Categories
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
after_initialize :rocketjob_categories_assign, if: :new_record?
|
10
|
+
after_initialize :rocketjob_categories_migrate, unless: :new_record?
|
11
|
+
before_perform :rocketjob_categories_input_render
|
12
|
+
after_perform :rocketjob_categories_output_render
|
13
|
+
|
14
|
+
# List of categories that this job can load input data into
|
15
|
+
embeds_many :input_categories, class_name: "RocketJob::Category::Input"
|
16
|
+
|
17
|
+
# List of categories that this job can save output data into
|
18
|
+
embeds_many :output_categories, class_name: "RocketJob::Category::Output"
|
19
|
+
|
20
|
+
# Internal attributes
|
21
|
+
class_attribute :defined_input_categories, instance_accessor: false, instance_predicate: false
|
22
|
+
class_attribute :defined_output_categories, instance_accessor: false, instance_predicate: false
|
23
|
+
|
24
|
+
# For RJMC to be able to edit jobs
|
25
|
+
accepts_nested_attributes_for :input_categories, :output_categories
|
26
|
+
end
|
27
|
+
|
28
|
+
module ClassMethods
|
29
|
+
# Define a new input category
|
30
|
+
# @see RocketJob::Category::Input
|
31
|
+
def input_category(**args)
|
32
|
+
category = RocketJob::Category::Input.new(**args)
|
33
|
+
if defined_input_categories.nil?
|
34
|
+
self.defined_input_categories = [category]
|
35
|
+
else
|
36
|
+
rocketjob_categories_set(category, defined_input_categories)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# Define a new output category
|
41
|
+
# @see RocketJob::Category::Output
|
42
|
+
def output_category(**args)
|
43
|
+
category = RocketJob::Category::Output.new(**args)
|
44
|
+
if defined_output_categories.nil?
|
45
|
+
self.defined_output_categories = [category]
|
46
|
+
else
|
47
|
+
rocketjob_categories_set(category, defined_output_categories)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Builds this job instance from the supplied properties hash that may contain input and output categories.
|
52
|
+
# Keeps the defaults and merges in settings without replacing existing categories.
|
53
|
+
def from_properties(properties)
|
54
|
+
return super(properties) unless properties.key?("input_categories") || properties.key?("output_categories")
|
55
|
+
|
56
|
+
properties = properties.dup
|
57
|
+
input_categories = properties.delete("input_categories")
|
58
|
+
output_categories = properties.delete("output_categories")
|
59
|
+
job = super(properties)
|
60
|
+
job.merge_input_categories(input_categories)
|
61
|
+
job.merge_output_categories(output_categories)
|
62
|
+
job
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def rocketjob_categories_set(category, categories)
|
68
|
+
index = categories.find_index { |cat| cat.name == category.name }
|
69
|
+
index ? categories[index] = category : categories << category
|
70
|
+
category
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def input_category(category_name = :main)
|
75
|
+
category_name = category_name.to_sym
|
76
|
+
category = nil
|
77
|
+
# .find does not work against this association
|
78
|
+
input_categories.each { |catg| category = catg if catg.name == category_name }
|
79
|
+
unless category
|
80
|
+
# Auto-register main input category if missing
|
81
|
+
if category_name == :main
|
82
|
+
category = Category::Input.new
|
83
|
+
self.input_categories = [category]
|
84
|
+
else
|
85
|
+
raise(ArgumentError,
|
86
|
+
"Unknown Input Category: #{category_name.inspect}. Registered categories: #{input_categories.collect(&:name).join(',')}")
|
87
|
+
end
|
88
|
+
end
|
89
|
+
category
|
90
|
+
end
|
91
|
+
|
92
|
+
def output_category(category_name = :main)
|
93
|
+
category_name = category_name.to_sym
|
94
|
+
category = nil
|
95
|
+
# .find does not work against this association
|
96
|
+
output_categories.each { |catg| category = catg if catg.name == category_name }
|
97
|
+
unless category
|
98
|
+
raise(ArgumentError,
|
99
|
+
"Unknown Output Category: #{category_name.inspect}. Registered categories: #{output_categories.collect(&:name).join(',')}")
|
100
|
+
end
|
101
|
+
|
102
|
+
category
|
103
|
+
end
|
104
|
+
|
105
|
+
# Returns [true|false] whether the named category has already been defined
|
106
|
+
def input_category?(category_name)
|
107
|
+
category_name = category_name.to_sym
|
108
|
+
# .find does not work against this association
|
109
|
+
input_categories.each { |catg| return true if catg.name == category_name }
|
110
|
+
false
|
111
|
+
end
|
112
|
+
|
113
|
+
def output_category?(category_name)
|
114
|
+
category_name = category_name.to_sym
|
115
|
+
# .find does not work against this association
|
116
|
+
output_categories.each { |catg| return true if catg.name == category_name }
|
117
|
+
false
|
118
|
+
end
|
119
|
+
|
120
|
+
def merge_input_categories(categories)
|
121
|
+
return if categories.blank?
|
122
|
+
|
123
|
+
categories.each do |properties|
|
124
|
+
category_name = (properties["name"] || properties[:name] || :main).to_sym
|
125
|
+
category = input_category(category_name)
|
126
|
+
properties.each { |key, value| category.public_send("#{key}=".to_sym, value) }
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def merge_output_categories(categories)
|
131
|
+
return if categories.blank?
|
132
|
+
|
133
|
+
categories.each do |properties|
|
134
|
+
category_name = (properties["name"] || properties[:name] || :main).to_sym
|
135
|
+
category = output_category(category_name)
|
136
|
+
properties.each { |key, value| category.public_send("#{key}=".to_sym, value) }
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
private
|
141
|
+
|
142
|
+
def rocketjob_categories_assign
|
143
|
+
# Input categories defaults to :main if none was set in the class
|
144
|
+
if input_categories.empty?
|
145
|
+
self.input_categories =
|
146
|
+
if self.class.defined_input_categories
|
147
|
+
self.class.defined_input_categories.deep_dup
|
148
|
+
else
|
149
|
+
[RocketJob::Category::Input.new]
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
return if !self.class.defined_output_categories || !output_categories.empty?
|
154
|
+
|
155
|
+
# Input categories defaults to nil if none was set in the class
|
156
|
+
self.output_categories = self.class.defined_output_categories.deep_dup
|
157
|
+
end
|
158
|
+
|
159
|
+
# Render the output from the perform.
|
160
|
+
def rocketjob_categories_output_render
|
161
|
+
return if @rocket_job_output.nil?
|
162
|
+
|
163
|
+
# TODO: ..
|
164
|
+
return unless output_categories
|
165
|
+
return if output_categories.empty?
|
166
|
+
|
167
|
+
@rocket_job_output = rocketjob_categories_output_render_row(@rocket_job_output)
|
168
|
+
end
|
169
|
+
|
170
|
+
# Parse the input data before passing to the perform method
|
171
|
+
def rocketjob_categories_input_render
|
172
|
+
return if @rocket_job_input.nil?
|
173
|
+
|
174
|
+
@rocket_job_input = rocketjob_categories_input_render_row(@rocket_job_input)
|
175
|
+
end
|
176
|
+
|
177
|
+
def rocketjob_categories_input_render_row(row)
|
178
|
+
return if row.nil?
|
179
|
+
|
180
|
+
category = input_category
|
181
|
+
return row if category.nil? || !category.tabular?
|
182
|
+
return nil if row.blank?
|
183
|
+
|
184
|
+
tabular = category.tabular
|
185
|
+
|
186
|
+
# Return the row as-is if the required header has not yet been set.
|
187
|
+
if tabular.header?
|
188
|
+
raise(ArgumentError,
|
189
|
+
"The tabular header columns _must_ be set before attempting to parse data that requires it.")
|
190
|
+
end
|
191
|
+
|
192
|
+
tabular.record_parse(row)
|
193
|
+
end
|
194
|
+
|
195
|
+
def rocketjob_categories_output_render_row(row)
|
196
|
+
return if row.nil?
|
197
|
+
|
198
|
+
if row.is_a?(Batch::Result)
|
199
|
+
category = output_category(row.category)
|
200
|
+
row.value = category.tabular.render(row.value) if category.tabular?
|
201
|
+
return row
|
202
|
+
end
|
203
|
+
|
204
|
+
if row.is_a?(Batch::Results)
|
205
|
+
results = Batch::Results.new
|
206
|
+
row.each { |result| results << rocketjob_categories_output_render_row(result) }
|
207
|
+
return results
|
208
|
+
end
|
209
|
+
|
210
|
+
category = output_category
|
211
|
+
return row unless category.tabular?
|
212
|
+
return nil if row.blank?
|
213
|
+
|
214
|
+
category.tabular.render(row)
|
215
|
+
end
|
216
|
+
|
217
|
+
# Migrate existing v4 batch jobs to v5.0
|
218
|
+
def rocketjob_categories_migrate
|
219
|
+
return unless attribute_present?(:input_categories) && self[:input_categories]&.first.is_a?(Symbol)
|
220
|
+
|
221
|
+
serializer = :none
|
222
|
+
if attribute_present?(:compress)
|
223
|
+
serializer = :compress if self[:compress]
|
224
|
+
remove_attribute(:compress)
|
225
|
+
end
|
226
|
+
|
227
|
+
if attribute_present?(:encrypt)
|
228
|
+
serializer = :encrypt if self[:encrypt]
|
229
|
+
remove_attribute(:encrypt)
|
230
|
+
end
|
231
|
+
|
232
|
+
slice_size = 100
|
233
|
+
if attribute_present?(:slice_size)
|
234
|
+
slice_size = self[:slice_size].to_i
|
235
|
+
remove_attribute(:slice_size)
|
236
|
+
end
|
237
|
+
|
238
|
+
main_input_format = nil
|
239
|
+
main_input_mode = :line
|
240
|
+
main_input_columns = nil
|
241
|
+
# Only migrate tabular attributes if the job also removed the tabular plugin.
|
242
|
+
unless respond_to?(:tabular_input_render)
|
243
|
+
if attribute_present?(:tabular_input_format)
|
244
|
+
main_input_format = self[:tabular_input_format]
|
245
|
+
remove_attribute(:tabular_input_format)
|
246
|
+
end
|
247
|
+
|
248
|
+
if attribute_present?(:tabular_input_mode)
|
249
|
+
main_input_mode = self[:tabular_input_mode]
|
250
|
+
remove_attribute(:tabular_input_mode)
|
251
|
+
end
|
252
|
+
|
253
|
+
if attribute_present?(:tabular_input_header)
|
254
|
+
main_input_columns = self[:tabular_input_header]
|
255
|
+
remove_attribute(:tabular_input_header)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
file_name = nil
|
260
|
+
if attribute_present?(:upload_file_name)
|
261
|
+
file_name = self[:upload_file_name]
|
262
|
+
remove_attribute(:upload_file_name)
|
263
|
+
end
|
264
|
+
|
265
|
+
existing = self[:input_categories]
|
266
|
+
self[:input_categories] = []
|
267
|
+
self[:input_categories] = existing.collect do |category_name|
|
268
|
+
RocketJob::Category::Input.new(
|
269
|
+
name: category_name,
|
270
|
+
file_name: file_name,
|
271
|
+
serializer: serializer,
|
272
|
+
slice_size: slice_size,
|
273
|
+
format: [:main, "main"].include?(category_name) ? main_input_format : nil,
|
274
|
+
columns: [:main, "main"].include?(category_name) ? main_input_columns : nil,
|
275
|
+
mode: [:main, "main"].include?(category_name) ? main_input_mode : nil
|
276
|
+
).as_document
|
277
|
+
end
|
278
|
+
|
279
|
+
collect_output = false
|
280
|
+
if attribute_present?(:collect_output)
|
281
|
+
collect_output = self[:collect_output]
|
282
|
+
remove_attribute(:collect_output)
|
283
|
+
end
|
284
|
+
|
285
|
+
collect_nil_output = true
|
286
|
+
if attribute_present?(:collect_nil_output)
|
287
|
+
collect_nil_output = self[:collect_nil_output]
|
288
|
+
remove_attribute(:collect_nil_output)
|
289
|
+
end
|
290
|
+
|
291
|
+
main_output_format = nil
|
292
|
+
main_output_columns = nil
|
293
|
+
main_output_options = nil
|
294
|
+
|
295
|
+
# Only migrate tabular attributes if the job also removed the tabular plugin.
|
296
|
+
unless respond_to?(:tabular_output_render)
|
297
|
+
if attribute_present?(:tabular_output_format)
|
298
|
+
main_output_format = self[:tabular_output_format]
|
299
|
+
remove_attribute(:tabular_output_format)
|
300
|
+
end
|
301
|
+
|
302
|
+
if attribute_present?(:tabular_output_header)
|
303
|
+
main_output_columns = self[:tabular_output_header]
|
304
|
+
remove_attribute(:tabular_output_header)
|
305
|
+
end
|
306
|
+
|
307
|
+
if attribute_present?(:tabular_output_options)
|
308
|
+
main_output_options = self[:tabular_output_options]
|
309
|
+
remove_attribute(:tabular_output_options)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
existing = self[:output_categories]
|
314
|
+
self[:output_categories] = []
|
315
|
+
if collect_output
|
316
|
+
if existing.blank?
|
317
|
+
self[:output_categories] = [
|
318
|
+
RocketJob::Category::Output.new(
|
319
|
+
nils: collect_nil_output,
|
320
|
+
format: main_output_format,
|
321
|
+
columns: main_output_columns,
|
322
|
+
format_options: main_output_options
|
323
|
+
).as_document
|
324
|
+
]
|
325
|
+
elsif existing.first.is_a?(Symbol)
|
326
|
+
self[:output_categories] = existing.collect do |category_name|
|
327
|
+
RocketJob::Category::Output.new(
|
328
|
+
name: category_name,
|
329
|
+
serializer: serializer,
|
330
|
+
nils: collect_nil_output,
|
331
|
+
format: [:main, "main"].include?(category_name) ? main_output_format : nil,
|
332
|
+
columns: [:main, "main"].include?(category_name) ? main_output_columns : nil,
|
333
|
+
format_options: [:main, "main"].include?(category_name) ? main_output_options : nil
|
334
|
+
).as_document
|
335
|
+
end
|
336
|
+
end
|
337
|
+
end
|
338
|
+
end
|
339
|
+
end
|
340
|
+
end
|
341
|
+
end
|