rocketjob 5.4.1 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +175 -5
- data/bin/rocketjob_batch_perf +1 -1
- data/bin/rocketjob_perf +1 -1
- data/lib/rocket_job/batch/categories.rb +345 -0
- data/lib/rocket_job/batch/io.rb +174 -106
- data/lib/rocket_job/batch/model.rb +20 -68
- data/lib/rocket_job/batch/performance.rb +19 -7
- data/lib/rocket_job/batch/statistics.rb +34 -12
- data/lib/rocket_job/batch/throttle_running_workers.rb +2 -6
- data/lib/rocket_job/batch/worker.rb +31 -26
- data/lib/rocket_job/batch.rb +3 -1
- data/lib/rocket_job/category/base.rb +81 -0
- data/lib/rocket_job/category/input.rb +170 -0
- data/lib/rocket_job/category/output.rb +34 -0
- data/lib/rocket_job/cli.rb +25 -17
- data/lib/rocket_job/dirmon_entry.rb +23 -13
- data/lib/rocket_job/event.rb +1 -1
- data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
- data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
- data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
- data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
- data/lib/rocket_job/jobs/conversion_job.rb +43 -0
- data/lib/rocket_job/jobs/dirmon_job.rb +25 -36
- data/lib/rocket_job/jobs/housekeeping_job.rb +11 -12
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +24 -11
- data/lib/rocket_job/jobs/on_demand_job.rb +3 -4
- data/lib/rocket_job/jobs/performance_job.rb +3 -1
- data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -96
- data/lib/rocket_job/jobs/upload_file_job.rb +48 -8
- data/lib/rocket_job/lookup_collection.rb +69 -0
- data/lib/rocket_job/plugins/cron.rb +60 -20
- data/lib/rocket_job/plugins/job/model.rb +25 -50
- data/lib/rocket_job/plugins/job/persistence.rb +36 -0
- data/lib/rocket_job/plugins/job/throttle.rb +2 -2
- data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +1 -1
- data/lib/rocket_job/plugins/job/worker.rb +2 -7
- data/lib/rocket_job/plugins/restart.rb +3 -103
- data/lib/rocket_job/plugins/state_machine.rb +4 -3
- data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +37 -0
- data/lib/rocket_job/ractor_worker.rb +42 -0
- data/lib/rocket_job/server/model.rb +1 -1
- data/lib/rocket_job/sliced/bzip2_output_slice.rb +18 -19
- data/lib/rocket_job/sliced/compressed_slice.rb +3 -6
- data/lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb +49 -0
- data/lib/rocket_job/sliced/encrypted_slice.rb +4 -6
- data/lib/rocket_job/sliced/input.rb +42 -54
- data/lib/rocket_job/sliced/slice.rb +12 -16
- data/lib/rocket_job/sliced/slices.rb +26 -11
- data/lib/rocket_job/sliced/writer/input.rb +46 -18
- data/lib/rocket_job/sliced/writer/output.rb +33 -45
- data/lib/rocket_job/sliced.rb +1 -74
- data/lib/rocket_job/subscribers/server.rb +1 -1
- data/lib/rocket_job/thread_worker.rb +46 -0
- data/lib/rocket_job/throttle_definitions.rb +7 -1
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +21 -55
- data/lib/rocket_job/worker_pool.rb +5 -7
- data/lib/rocketjob.rb +53 -43
- metadata +36 -28
- data/lib/rocket_job/batch/tabular/input.rb +0 -131
- data/lib/rocket_job/batch/tabular/output.rb +0 -65
- data/lib/rocket_job/batch/tabular.rb +0 -56
- data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
- data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
@@ -1,65 +0,0 @@
|
|
1
|
-
require "active_support/concern"
|
2
|
-
|
3
|
-
module RocketJob
|
4
|
-
module Batch
|
5
|
-
class Tabular
|
6
|
-
# For the simple case where all `output_categories` have the same format,
|
7
|
-
# If multiple output categories are used with different formats, then use IOStreams::Tabular directly
|
8
|
-
# instead of this plugin.
|
9
|
-
module Output
|
10
|
-
extend ActiveSupport::Concern
|
11
|
-
|
12
|
-
included do
|
13
|
-
field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
|
14
|
-
field :tabular_output_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
|
15
|
-
field :tabular_output_options, type: Hash, class_attribute: true
|
16
|
-
|
17
|
-
validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
|
18
|
-
|
19
|
-
after_perform :tabular_output_render
|
20
|
-
end
|
21
|
-
|
22
|
-
# Clear out cached tabular_output any time header or format is changed.
|
23
|
-
def tabular_output_header=(tabular_output_header)
|
24
|
-
super(tabular_output_header)
|
25
|
-
@tabular_output = nil
|
26
|
-
end
|
27
|
-
|
28
|
-
def tabular_output_format=(tabular_output_format)
|
29
|
-
super(tabular_output_format)
|
30
|
-
@tabular_output = nil
|
31
|
-
end
|
32
|
-
|
33
|
-
# Overrides: `RocketJob::Batch::IO#download` to add the `tabular_output_header`.
|
34
|
-
def download(file_name_or_io = nil, category: :main, **args, &block)
|
35
|
-
unless tabular_output.requires_header?(category)
|
36
|
-
return super(file_name_or_io, category: category, **args, &block)
|
37
|
-
end
|
38
|
-
|
39
|
-
header = tabular_output.render_header(category)
|
40
|
-
super(file_name_or_io, header_line: header, category: category, **args, &block)
|
41
|
-
end
|
42
|
-
|
43
|
-
private
|
44
|
-
|
45
|
-
# Delimited instance used for this slice, by a single worker (thread)
|
46
|
-
def tabular_output
|
47
|
-
@tabular_output ||= Tabular.new(
|
48
|
-
main: IOStreams::Tabular.new(
|
49
|
-
columns: tabular_output_header,
|
50
|
-
format: tabular_output_format,
|
51
|
-
format_options: tabular_output_options&.deep_symbolize_keys
|
52
|
-
)
|
53
|
-
)
|
54
|
-
end
|
55
|
-
|
56
|
-
# Render the output from the perform.
|
57
|
-
def tabular_output_render
|
58
|
-
return unless collect_output?
|
59
|
-
|
60
|
-
@rocket_job_output = tabular_output.render(@rocket_job_output)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
@@ -1,56 +0,0 @@
|
|
1
|
-
module RocketJob
|
2
|
-
module Batch
|
3
|
-
# Format output results.
|
4
|
-
#
|
5
|
-
# Takes Batch::Results, Batch::Result, Hash, Array, or String and renders it for output.
|
6
|
-
#
|
7
|
-
# Example:
|
8
|
-
#
|
9
|
-
# tabular = Tabular.new(
|
10
|
-
# main: IOStreams::Tabular.new(columns: main_file_headers, format: tabular_output_format),
|
11
|
-
# exceptions: IOStreams::Tabular.new(columns: exception_file_headers, format: tabular_output_format)
|
12
|
-
# )
|
13
|
-
#
|
14
|
-
# tabular.render(row)
|
15
|
-
class Tabular
|
16
|
-
autoload :Input, "rocket_job/batch/tabular/input"
|
17
|
-
autoload :Output, "rocket_job/batch/tabular/output"
|
18
|
-
|
19
|
-
def initialize(map)
|
20
|
-
@map = map
|
21
|
-
end
|
22
|
-
|
23
|
-
def [](category = :main)
|
24
|
-
@map[category] || raise("No tabular map defined for category: #{category.inspect}")
|
25
|
-
end
|
26
|
-
|
27
|
-
# Iterate over responses and format using Tabular
|
28
|
-
def render(row, category = :main)
|
29
|
-
if row.is_a?(Batch::Results)
|
30
|
-
results = Batch::Results.new
|
31
|
-
row.each { |result| results << render(result) }
|
32
|
-
results
|
33
|
-
elsif row.is_a?(Batch::Result)
|
34
|
-
row.value = self[row.category].render(row.value)
|
35
|
-
row
|
36
|
-
elsif row.blank?
|
37
|
-
nil
|
38
|
-
else
|
39
|
-
self[category].render(row)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def render_header(category = :main)
|
44
|
-
self[category].render_header
|
45
|
-
end
|
46
|
-
|
47
|
-
def requires_header?(category = :main)
|
48
|
-
self[category].requires_header?
|
49
|
-
end
|
50
|
-
|
51
|
-
def header?(category = :main)
|
52
|
-
self[category].header?
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
require "mongoid/fields/validators/macro"
|
2
|
-
require "semantic_logger"
|
3
|
-
module RocketJob
|
4
|
-
module RemoveMongoidWarnings
|
5
|
-
# Remove annoying warnings about Symbols type being deprecated.
|
6
|
-
def validate_options(*params)
|
7
|
-
SemanticLogger.silence(:error) { super(*params) }
|
8
|
-
end
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
|
@@ -1,28 +0,0 @@
|
|
1
|
-
# Job to dynamically perform ruby code on demand as a Batch,
|
2
|
-
# with input and/or output from CSV/JSON or other format supported by Tabular.
|
3
|
-
#
|
4
|
-
# Nodes:
|
5
|
-
# - Need to specify `destroy_on_complete: false` to collect output from this job.
|
6
|
-
# - `after_code` can be used to automatically download the output of this job to a file on completion.
|
7
|
-
#
|
8
|
-
# Example: Iterate over all rows in a table:
|
9
|
-
# code = <<-CODE
|
10
|
-
# if user = User.find(row)
|
11
|
-
# user.cleanse_attributes!
|
12
|
-
# user.save(validate: false)
|
13
|
-
# end
|
14
|
-
# CODE
|
15
|
-
# job = RocketJob::Jobs::OnDemandBatchTabularJob.new(code: code, description: 'cleanse users', destroy_on_complete: false)
|
16
|
-
# job.upload("users.csv")
|
17
|
-
# job.save!
|
18
|
-
#
|
19
|
-
# On completion export the output:
|
20
|
-
# job.download("output.csv")
|
21
|
-
module RocketJob
|
22
|
-
module Jobs
|
23
|
-
class OnDemandBatchTabularJob < OnDemandBatchJob
|
24
|
-
include RocketJob::Batch::Tabular::Input
|
25
|
-
include RocketJob::Batch::Tabular::Output
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|