rocketjob 5.4.1 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +175 -5
  3. data/bin/rocketjob_batch_perf +1 -1
  4. data/bin/rocketjob_perf +1 -1
  5. data/lib/rocket_job/batch/categories.rb +345 -0
  6. data/lib/rocket_job/batch/io.rb +174 -106
  7. data/lib/rocket_job/batch/model.rb +20 -68
  8. data/lib/rocket_job/batch/performance.rb +19 -7
  9. data/lib/rocket_job/batch/statistics.rb +34 -12
  10. data/lib/rocket_job/batch/throttle_running_workers.rb +2 -6
  11. data/lib/rocket_job/batch/worker.rb +31 -26
  12. data/lib/rocket_job/batch.rb +3 -1
  13. data/lib/rocket_job/category/base.rb +81 -0
  14. data/lib/rocket_job/category/input.rb +170 -0
  15. data/lib/rocket_job/category/output.rb +34 -0
  16. data/lib/rocket_job/cli.rb +25 -17
  17. data/lib/rocket_job/dirmon_entry.rb +23 -13
  18. data/lib/rocket_job/event.rb +1 -1
  19. data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
  20. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
  21. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
  22. data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
  23. data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
  24. data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
  25. data/lib/rocket_job/jobs/conversion_job.rb +43 -0
  26. data/lib/rocket_job/jobs/dirmon_job.rb +25 -36
  27. data/lib/rocket_job/jobs/housekeeping_job.rb +11 -12
  28. data/lib/rocket_job/jobs/on_demand_batch_job.rb +24 -11
  29. data/lib/rocket_job/jobs/on_demand_job.rb +3 -4
  30. data/lib/rocket_job/jobs/performance_job.rb +3 -1
  31. data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -96
  32. data/lib/rocket_job/jobs/upload_file_job.rb +48 -8
  33. data/lib/rocket_job/lookup_collection.rb +69 -0
  34. data/lib/rocket_job/plugins/cron.rb +60 -20
  35. data/lib/rocket_job/plugins/job/model.rb +25 -50
  36. data/lib/rocket_job/plugins/job/persistence.rb +36 -0
  37. data/lib/rocket_job/plugins/job/throttle.rb +2 -2
  38. data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +1 -1
  39. data/lib/rocket_job/plugins/job/worker.rb +2 -7
  40. data/lib/rocket_job/plugins/restart.rb +3 -103
  41. data/lib/rocket_job/plugins/state_machine.rb +4 -3
  42. data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +37 -0
  43. data/lib/rocket_job/ractor_worker.rb +42 -0
  44. data/lib/rocket_job/server/model.rb +1 -1
  45. data/lib/rocket_job/sliced/bzip2_output_slice.rb +18 -19
  46. data/lib/rocket_job/sliced/compressed_slice.rb +3 -6
  47. data/lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb +49 -0
  48. data/lib/rocket_job/sliced/encrypted_slice.rb +4 -6
  49. data/lib/rocket_job/sliced/input.rb +42 -54
  50. data/lib/rocket_job/sliced/slice.rb +12 -16
  51. data/lib/rocket_job/sliced/slices.rb +26 -11
  52. data/lib/rocket_job/sliced/writer/input.rb +46 -18
  53. data/lib/rocket_job/sliced/writer/output.rb +33 -45
  54. data/lib/rocket_job/sliced.rb +1 -74
  55. data/lib/rocket_job/subscribers/server.rb +1 -1
  56. data/lib/rocket_job/thread_worker.rb +46 -0
  57. data/lib/rocket_job/throttle_definitions.rb +7 -1
  58. data/lib/rocket_job/version.rb +1 -1
  59. data/lib/rocket_job/worker.rb +21 -55
  60. data/lib/rocket_job/worker_pool.rb +5 -7
  61. data/lib/rocketjob.rb +53 -43
  62. metadata +36 -28
  63. data/lib/rocket_job/batch/tabular/input.rb +0 -131
  64. data/lib/rocket_job/batch/tabular/output.rb +0 -65
  65. data/lib/rocket_job/batch/tabular.rb +0 -56
  66. data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
  67. data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
@@ -1,65 +0,0 @@
1
- require "active_support/concern"
2
-
3
- module RocketJob
4
- module Batch
5
- class Tabular
6
- # For the simple case where all `output_categories` have the same format,
7
- # If multiple output categories are used with different formats, then use IOStreams::Tabular directly
8
- # instead of this plugin.
9
- module Output
10
- extend ActiveSupport::Concern
11
-
12
- included do
13
- field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
14
- field :tabular_output_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
15
- field :tabular_output_options, type: Hash, class_attribute: true
16
-
17
- validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
18
-
19
- after_perform :tabular_output_render
20
- end
21
-
22
- # Clear out cached tabular_output any time header or format is changed.
23
- def tabular_output_header=(tabular_output_header)
24
- super(tabular_output_header)
25
- @tabular_output = nil
26
- end
27
-
28
- def tabular_output_format=(tabular_output_format)
29
- super(tabular_output_format)
30
- @tabular_output = nil
31
- end
32
-
33
- # Overrides: `RocketJob::Batch::IO#download` to add the `tabular_output_header`.
34
- def download(file_name_or_io = nil, category: :main, **args, &block)
35
- unless tabular_output.requires_header?(category)
36
- return super(file_name_or_io, category: category, **args, &block)
37
- end
38
-
39
- header = tabular_output.render_header(category)
40
- super(file_name_or_io, header_line: header, category: category, **args, &block)
41
- end
42
-
43
- private
44
-
45
- # Delimited instance used for this slice, by a single worker (thread)
46
- def tabular_output
47
- @tabular_output ||= Tabular.new(
48
- main: IOStreams::Tabular.new(
49
- columns: tabular_output_header,
50
- format: tabular_output_format,
51
- format_options: tabular_output_options&.deep_symbolize_keys
52
- )
53
- )
54
- end
55
-
56
- # Render the output from the perform.
57
- def tabular_output_render
58
- return unless collect_output?
59
-
60
- @rocket_job_output = tabular_output.render(@rocket_job_output)
61
- end
62
- end
63
- end
64
- end
65
- end
@@ -1,56 +0,0 @@
1
- module RocketJob
2
- module Batch
3
- # Format output results.
4
- #
5
- # Takes Batch::Results, Batch::Result, Hash, Array, or String and renders it for output.
6
- #
7
- # Example:
8
- #
9
- # tabular = Tabular.new(
10
- # main: IOStreams::Tabular.new(columns: main_file_headers, format: tabular_output_format),
11
- # exceptions: IOStreams::Tabular.new(columns: exception_file_headers, format: tabular_output_format)
12
- # )
13
- #
14
- # tabular.render(row)
15
- class Tabular
16
- autoload :Input, "rocket_job/batch/tabular/input"
17
- autoload :Output, "rocket_job/batch/tabular/output"
18
-
19
- def initialize(map)
20
- @map = map
21
- end
22
-
23
- def [](category = :main)
24
- @map[category] || raise("No tabular map defined for category: #{category.inspect}")
25
- end
26
-
27
- # Iterate over responses and format using Tabular
28
- def render(row, category = :main)
29
- if row.is_a?(Batch::Results)
30
- results = Batch::Results.new
31
- row.each { |result| results << render(result) }
32
- results
33
- elsif row.is_a?(Batch::Result)
34
- row.value = self[row.category].render(row.value)
35
- row
36
- elsif row.blank?
37
- nil
38
- else
39
- self[category].render(row)
40
- end
41
- end
42
-
43
- def render_header(category = :main)
44
- self[category].render_header
45
- end
46
-
47
- def requires_header?(category = :main)
48
- self[category].requires_header?
49
- end
50
-
51
- def header?(category = :main)
52
- self[category].header?
53
- end
54
- end
55
- end
56
- end
@@ -1,12 +0,0 @@
1
- require "mongoid/fields/validators/macro"
2
- require "semantic_logger"
3
- module RocketJob
4
- module RemoveMongoidWarnings
5
- # Remove annoying warnings about Symbols type being deprecated.
6
- def validate_options(*params)
7
- SemanticLogger.silence(:error) { super(*params) }
8
- end
9
- end
10
- end
11
-
12
- ::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
@@ -1,28 +0,0 @@
1
- # Job to dynamically perform ruby code on demand as a Batch,
2
- # with input and/or output from CSV/JSON or other format supported by Tabular.
3
- #
4
- # Nodes:
5
- # - Need to specify `destroy_on_complete: false` to collect output from this job.
6
- # - `after_code` can be used to automatically download the output of this job to a file on completion.
7
- #
8
- # Example: Iterate over all rows in a table:
9
- # code = <<-CODE
10
- # if user = User.find(row)
11
- # user.cleanse_attributes!
12
- # user.save(validate: false)
13
- # end
14
- # CODE
15
- # job = RocketJob::Jobs::OnDemandBatchTabularJob.new(code: code, description: 'cleanse users', destroy_on_complete: false)
16
- # job.upload("users.csv")
17
- # job.save!
18
- #
19
- # On completion export the output:
20
- # job.download("output.csv")
21
- module RocketJob
22
- module Jobs
23
- class OnDemandBatchTabularJob < OnDemandBatchJob
24
- include RocketJob::Batch::Tabular::Input
25
- include RocketJob::Batch::Tabular::Output
26
- end
27
- end
28
- end