rocketjob 5.4.0.beta2 → 6.0.0.rc3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +149 -5
  3. data/bin/rocketjob_batch_perf +1 -1
  4. data/bin/rocketjob_perf +1 -1
  5. data/lib/rocket_job/batch.rb +3 -1
  6. data/lib/rocket_job/batch/categories.rb +341 -0
  7. data/lib/rocket_job/batch/io.rb +128 -60
  8. data/lib/rocket_job/batch/model.rb +20 -68
  9. data/lib/rocket_job/batch/performance.rb +19 -7
  10. data/lib/rocket_job/batch/statistics.rb +34 -12
  11. data/lib/rocket_job/batch/tabular.rb +2 -0
  12. data/lib/rocket_job/batch/tabular/input.rb +8 -6
  13. data/lib/rocket_job/batch/tabular/output.rb +4 -2
  14. data/lib/rocket_job/batch/throttle_running_workers.rb +8 -17
  15. data/lib/rocket_job/batch/worker.rb +27 -24
  16. data/lib/rocket_job/category/base.rb +78 -0
  17. data/lib/rocket_job/category/input.rb +110 -0
  18. data/lib/rocket_job/category/output.rb +25 -0
  19. data/lib/rocket_job/cli.rb +25 -17
  20. data/lib/rocket_job/dirmon_entry.rb +22 -12
  21. data/lib/rocket_job/event.rb +1 -1
  22. data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
  23. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
  24. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
  25. data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
  26. data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
  27. data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
  28. data/lib/rocket_job/jobs/conversion_job.rb +39 -0
  29. data/lib/rocket_job/jobs/dirmon_job.rb +2 -2
  30. data/lib/rocket_job/jobs/housekeeping_job.rb +7 -7
  31. data/lib/rocket_job/jobs/on_demand_batch_job.rb +17 -6
  32. data/lib/rocket_job/jobs/on_demand_job.rb +1 -2
  33. data/lib/rocket_job/jobs/performance_job.rb +3 -1
  34. data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -96
  35. data/lib/rocket_job/jobs/upload_file_job.rb +44 -8
  36. data/lib/rocket_job/lookup_collection.rb +69 -0
  37. data/lib/rocket_job/plugins/job/model.rb +25 -50
  38. data/lib/rocket_job/plugins/job/throttle.rb +2 -2
  39. data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +12 -4
  40. data/lib/rocket_job/plugins/job/worker.rb +2 -7
  41. data/lib/rocket_job/plugins/restart.rb +12 -5
  42. data/lib/rocket_job/plugins/state_machine.rb +2 -1
  43. data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +38 -0
  44. data/lib/rocket_job/ractor_worker.rb +42 -0
  45. data/lib/rocket_job/server/model.rb +1 -1
  46. data/lib/rocket_job/sliced.rb +15 -70
  47. data/lib/rocket_job/sliced/bzip2_output_slice.rb +1 -1
  48. data/lib/rocket_job/sliced/input.rb +1 -1
  49. data/lib/rocket_job/sliced/slice.rb +5 -13
  50. data/lib/rocket_job/sliced/slices.rb +14 -2
  51. data/lib/rocket_job/sliced/writer/output.rb +33 -45
  52. data/lib/rocket_job/subscribers/server.rb +1 -1
  53. data/lib/rocket_job/thread_worker.rb +46 -0
  54. data/lib/rocket_job/throttle_definitions.rb +7 -1
  55. data/lib/rocket_job/version.rb +1 -1
  56. data/lib/rocket_job/worker.rb +21 -55
  57. data/lib/rocket_job/worker_pool.rb +5 -7
  58. data/lib/rocketjob.rb +53 -43
  59. metadata +36 -26
  60. data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
  61. data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
data/lib/rocket_job/extensions/mongoid/remove_warnings.rb DELETED
@@ -1,12 +0,0 @@
1
- require "mongoid/fields/validators/macro"
2
- require "semantic_logger"
3
- module RocketJob
4
- module RemoveMongoidWarnings
5
- # Remove annoying warnings about Symbols type being deprecated.
6
- def validate_options(*params)
7
- SemanticLogger.silence(:error) { super(*params) }
8
- end
9
- end
10
- end
11
-
12
- ::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb DELETED
@@ -1,28 +0,0 @@
1
- # Job to dynamically perform ruby code on demand as a Batch,
2
- # with input and/or output from CSV/JSON or other format supported by Tabular.
3
- #
4
- # Nodes:
5
- # - Need to specify `destroy_on_complete: false` to collect output from this job.
6
- # - `after_code` can be used to automatically download the output of this job to a file on completion.
7
- #
8
- # Example: Iterate over all rows in a table:
9
- # code = <<-CODE
10
- # if user = User.find(row)
11
- # user.cleanse_attributes!
12
- # user.save(validate: false)
13
- # end
14
- # CODE
15
- # job = RocketJob::Jobs::OnDemandBatchTabularJob.new(code: code, description: 'cleanse users', destroy_on_complete: false)
16
- # job.upload("users.csv")
17
- # job.save!
18
- #
19
- # On completion export the output:
20
- # job.download("output.csv")
21
- module RocketJob
22
- module Jobs
23
- class OnDemandBatchTabularJob < OnDemandBatchJob
24
- include RocketJob::Batch::Tabular::Input
25
- include RocketJob::Batch::Tabular::Output
26
- end
27
- end
28
- end