rocketjob 5.4.1 → 6.0.0.rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +19 -5
  3. data/bin/rocketjob_batch_perf +1 -1
  4. data/bin/rocketjob_perf +1 -1
  5. data/lib/rocket_job/batch.rb +3 -0
  6. data/lib/rocket_job/batch/categories.rb +338 -0
  7. data/lib/rocket_job/batch/io.rb +128 -60
  8. data/lib/rocket_job/batch/model.rb +20 -68
  9. data/lib/rocket_job/batch/performance.rb +20 -8
  10. data/lib/rocket_job/batch/statistics.rb +35 -13
  11. data/lib/rocket_job/batch/tabular.rb +2 -0
  12. data/lib/rocket_job/batch/tabular/input.rb +8 -6
  13. data/lib/rocket_job/batch/tabular/output.rb +4 -2
  14. data/lib/rocket_job/batch/throttle_running_workers.rb +1 -5
  15. data/lib/rocket_job/batch/worker.rb +27 -24
  16. data/lib/rocket_job/category/base.rb +78 -0
  17. data/lib/rocket_job/category/input.rb +110 -0
  18. data/lib/rocket_job/category/output.rb +25 -0
  19. data/lib/rocket_job/cli.rb +24 -16
  20. data/lib/rocket_job/dirmon_entry.rb +22 -12
  21. data/lib/rocket_job/event.rb +1 -1
  22. data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
  23. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
  24. data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
  25. data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
  26. data/lib/rocket_job/jobs/dirmon_job.rb +1 -1
  27. data/lib/rocket_job/jobs/housekeeping_job.rb +7 -7
  28. data/lib/rocket_job/jobs/on_demand_batch_job.rb +15 -6
  29. data/lib/rocket_job/jobs/on_demand_job.rb +1 -2
  30. data/lib/rocket_job/jobs/performance_job.rb +3 -1
  31. data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +5 -4
  32. data/lib/rocket_job/jobs/upload_file_job.rb +46 -9
  33. data/lib/rocket_job/lookup_collection.rb +68 -0
  34. data/lib/rocket_job/plugins/job/model.rb +25 -50
  35. data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +1 -1
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -7
  37. data/lib/rocket_job/plugins/restart.rb +12 -5
  38. data/lib/rocket_job/plugins/state_machine.rb +2 -1
  39. data/lib/rocket_job/ractor_worker.rb +42 -0
  40. data/lib/rocket_job/server/model.rb +1 -1
  41. data/lib/rocket_job/sliced.rb +15 -70
  42. data/lib/rocket_job/sliced/input.rb +1 -1
  43. data/lib/rocket_job/sliced/slice.rb +5 -13
  44. data/lib/rocket_job/sliced/slices.rb +14 -2
  45. data/lib/rocket_job/sliced/writer/output.rb +33 -44
  46. data/lib/rocket_job/subscribers/server.rb +1 -1
  47. data/lib/rocket_job/thread_worker.rb +46 -0
  48. data/lib/rocket_job/version.rb +1 -1
  49. data/lib/rocket_job/worker.rb +21 -55
  50. data/lib/rocket_job/worker_pool.rb +5 -7
  51. data/lib/rocketjob.rb +52 -41
  52. metadata +35 -27
  53. data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
  54. data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
data/lib/rocket_job/extensions/mongoid/remove_warnings.rb DELETED
@@ -1,12 +0,0 @@
1
- require "mongoid/fields/validators/macro"
2
- require "semantic_logger"
3
- module RocketJob
4
- module RemoveMongoidWarnings
5
- # Remove annoying warnings about Symbols type being deprecated.
6
- def validate_options(*params)
7
- SemanticLogger.silence(:error) { super(*params) }
8
- end
9
- end
10
- end
11
-
12
- ::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb DELETED
@@ -1,28 +0,0 @@
1
- # Job to dynamically perform ruby code on demand as a Batch,
2
- # with input and/or output from CSV/JSON or other format supported by Tabular.
3
- #
4
- # Nodes:
5
- # - Need to specify `destroy_on_complete: false` to collect output from this job.
6
- # - `after_code` can be used to automatically download the output of this job to a file on completion.
7
- #
8
- # Example: Iterate over all rows in a table:
9
- # code = <<-CODE
10
- # if user = User.find(row)
11
- # user.cleanse_attributes!
12
- # user.save(validate: false)
13
- # end
14
- # CODE
15
- # job = RocketJob::Jobs::OnDemandBatchTabularJob.new(code: code, description: 'cleanse users', destroy_on_complete: false)
16
- # job.upload("users.csv")
17
- # job.save!
18
- #
19
- # On completion export the output:
20
- # job.download("output.csv")
21
- module RocketJob
22
- module Jobs
23
- class OnDemandBatchTabularJob < OnDemandBatchJob
24
- include RocketJob::Batch::Tabular::Input
25
- include RocketJob::Batch::Tabular::Output
26
- end
27
- end
28
- end