rocketjob 5.4.0.beta2 → 6.0.0.rc3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +149 -5
- data/bin/rocketjob_batch_perf +1 -1
- data/bin/rocketjob_perf +1 -1
- data/lib/rocket_job/batch.rb +3 -1
- data/lib/rocket_job/batch/categories.rb +341 -0
- data/lib/rocket_job/batch/io.rb +128 -60
- data/lib/rocket_job/batch/model.rb +20 -68
- data/lib/rocket_job/batch/performance.rb +19 -7
- data/lib/rocket_job/batch/statistics.rb +34 -12
- data/lib/rocket_job/batch/tabular.rb +2 -0
- data/lib/rocket_job/batch/tabular/input.rb +8 -6
- data/lib/rocket_job/batch/tabular/output.rb +4 -2
- data/lib/rocket_job/batch/throttle_running_workers.rb +8 -17
- data/lib/rocket_job/batch/worker.rb +27 -24
- data/lib/rocket_job/category/base.rb +78 -0
- data/lib/rocket_job/category/input.rb +110 -0
- data/lib/rocket_job/category/output.rb +25 -0
- data/lib/rocket_job/cli.rb +25 -17
- data/lib/rocket_job/dirmon_entry.rb +22 -12
- data/lib/rocket_job/event.rb +1 -1
- data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
- data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
- data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
- data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
- data/lib/rocket_job/jobs/conversion_job.rb +39 -0
- data/lib/rocket_job/jobs/dirmon_job.rb +2 -2
- data/lib/rocket_job/jobs/housekeeping_job.rb +7 -7
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +17 -6
- data/lib/rocket_job/jobs/on_demand_job.rb +1 -2
- data/lib/rocket_job/jobs/performance_job.rb +3 -1
- data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -96
- data/lib/rocket_job/jobs/upload_file_job.rb +44 -8
- data/lib/rocket_job/lookup_collection.rb +69 -0
- data/lib/rocket_job/plugins/job/model.rb +25 -50
- data/lib/rocket_job/plugins/job/throttle.rb +2 -2
- data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +12 -4
- data/lib/rocket_job/plugins/job/worker.rb +2 -7
- data/lib/rocket_job/plugins/restart.rb +12 -5
- data/lib/rocket_job/plugins/state_machine.rb +2 -1
- data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +38 -0
- data/lib/rocket_job/ractor_worker.rb +42 -0
- data/lib/rocket_job/server/model.rb +1 -1
- data/lib/rocket_job/sliced.rb +15 -70
- data/lib/rocket_job/sliced/bzip2_output_slice.rb +1 -1
- data/lib/rocket_job/sliced/input.rb +1 -1
- data/lib/rocket_job/sliced/slice.rb +5 -13
- data/lib/rocket_job/sliced/slices.rb +14 -2
- data/lib/rocket_job/sliced/writer/output.rb +33 -45
- data/lib/rocket_job/subscribers/server.rb +1 -1
- data/lib/rocket_job/thread_worker.rb +46 -0
- data/lib/rocket_job/throttle_definitions.rb +7 -1
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +21 -55
- data/lib/rocket_job/worker_pool.rb +5 -7
- data/lib/rocketjob.rb +53 -43
- metadata +36 -26
- data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
- data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
@@ -1,12 +0,0 @@
|
|
1
|
-
require "mongoid/fields/validators/macro"
|
2
|
-
require "semantic_logger"
|
3
|
-
module RocketJob
|
4
|
-
module RemoveMongoidWarnings
|
5
|
-
# Remove annoying warnings about Symbols type being deprecated.
|
6
|
-
def validate_options(*params)
|
7
|
-
SemanticLogger.silence(:error) { super(*params) }
|
8
|
-
end
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
|
@@ -1,28 +0,0 @@
|
|
1
|
-
# Job to dynamically perform ruby code on demand as a Batch,
|
2
|
-
# with input and/or output from CSV/JSON or other format supported by Tabular.
|
3
|
-
#
|
4
|
-
# Nodes:
|
5
|
-
# - Need to specify `destroy_on_complete: false` to collect output from this job.
|
6
|
-
# - `after_code` can be used to automatically download the output of this job to a file on completion.
|
7
|
-
#
|
8
|
-
# Example: Iterate over all rows in a table:
|
9
|
-
# code = <<-CODE
|
10
|
-
# if user = User.find(row)
|
11
|
-
# user.cleanse_attributes!
|
12
|
-
# user.save(validate: false)
|
13
|
-
# end
|
14
|
-
# CODE
|
15
|
-
# job = RocketJob::Jobs::OnDemandBatchTabularJob.new(code: code, description: 'cleanse users', destroy_on_complete: false)
|
16
|
-
# job.upload("users.csv")
|
17
|
-
# job.save!
|
18
|
-
#
|
19
|
-
# On completion export the output:
|
20
|
-
# job.download("output.csv")
|
21
|
-
module RocketJob
|
22
|
-
module Jobs
|
23
|
-
class OnDemandBatchTabularJob < OnDemandBatchJob
|
24
|
-
include RocketJob::Batch::Tabular::Input
|
25
|
-
include RocketJob::Batch::Tabular::Output
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|