sidekiq-web_custom 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +109 -0
  3. data/.gitignore +18 -0
  4. data/.rspec +3 -0
  5. data/CHANGELOG.md +5 -0
  6. data/CODE_OF_CONDUCT.md +84 -0
  7. data/Dockerfile +18 -0
  8. data/Gemfile +14 -0
  9. data/Gemfile.lock +182 -0
  10. data/LICENSE.txt +21 -0
  11. data/Makefile +40 -0
  12. data/README.md +92 -0
  13. data/Rakefile +8 -0
  14. data/bin/all_workers +69 -0
  15. data/bin/bundle +114 -0
  16. data/bin/publish +40 -0
  17. data/bin/publish_git +38 -0
  18. data/bin/publish_ruby_gems +22 -0
  19. data/bin/rails +18 -0
  20. data/bin/sidekiq +29 -0
  21. data/bin/sidekiqmon +29 -0
  22. data/docker-compose.yml +43 -0
  23. data/dummy_rails/.rspec +4 -0
  24. data/dummy_rails/app/assets/config/manifest.js +0 -0
  25. data/dummy_rails/app/controllers/application_controller.rb +2 -0
  26. data/dummy_rails/app/helpers/application_helper.rb +2 -0
  27. data/dummy_rails/app/views/layouts/application.html.erb +15 -0
  28. data/dummy_rails/app/views/layouts/mailer.html.erb +13 -0
  29. data/dummy_rails/app/workers/exclude_hard_worker.rb +15 -0
  30. data/dummy_rails/app/workers/exclude_lazy_worker.rb +11 -0
  31. data/dummy_rails/app/workers/exclude_random_raise_worker.rb +11 -0
  32. data/dummy_rails/app/workers/hard_worker.rb +15 -0
  33. data/dummy_rails/app/workers/lazy_worker.rb +10 -0
  34. data/dummy_rails/app/workers/random_raise_worker.rb +12 -0
  35. data/dummy_rails/app/workers/taco_worker.rb +12 -0
  36. data/dummy_rails/app/workers/tostada_worker.rb +11 -0
  37. data/dummy_rails/bin/rspec +5 -0
  38. data/dummy_rails/config.ru +5 -0
  39. data/dummy_rails/config/application.rb +20 -0
  40. data/dummy_rails/config/boot.rb +5 -0
  41. data/dummy_rails/config/environment.rb +5 -0
  42. data/dummy_rails/config/environments/development.rb +52 -0
  43. data/dummy_rails/config/environments/production.rb +88 -0
  44. data/dummy_rails/config/environments/test.rb +40 -0
  45. data/dummy_rails/config/initializers/assets.rb +14 -0
  46. data/dummy_rails/config/initializers/cookies_serializer.rb +5 -0
  47. data/dummy_rails/config/initializers/filter_parameter_logging.rb +4 -0
  48. data/dummy_rails/config/initializers/sidekiq-web_custom.rb +3 -0
  49. data/dummy_rails/config/initializers/wrap_parameters.rb +14 -0
  50. data/dummy_rails/config/locales/en.yml +33 -0
  51. data/dummy_rails/config/puma.rb +37 -0
  52. data/dummy_rails/config/routes.rb +5 -0
  53. data/dummy_rails/config/secrets.yml +5 -0
  54. data/dummy_rails/config/spring.rb +5 -0
  55. data/dummy_rails/config/storage.yml +34 -0
  56. data/dummy_rails/package.json +5 -0
  57. data/dummy_rails/spec/spec_helper.rb +50 -0
  58. data/lib/load_random_workers.rb +25 -0
  59. data/lib/sidekiq/views/actions/queues/delete.erb +12 -0
  60. data/lib/sidekiq/views/actions/queues/drain_queue.erb +4 -0
  61. data/lib/sidekiq/views/actions/queues/pause.erb +13 -0
  62. data/lib/sidekiq/views/actions/retries/delete.erb +5 -0
  63. data/lib/sidekiq/views/actions/retries/execute.erb +7 -0
  64. data/lib/sidekiq/views/actions/scheduled/delete.erb +5 -0
  65. data/lib/sidekiq/views/actions/scheduled/execute.erb +7 -0
  66. data/lib/sidekiq/views/queues.erb +64 -0
  67. data/lib/sidekiq/views/retries.erb +123 -0
  68. data/lib/sidekiq/views/scheduled.erb +97 -0
  69. data/lib/sidekiq/web_custom.rb +95 -0
  70. data/lib/sidekiq/web_custom/configuration.rb +112 -0
  71. data/lib/sidekiq/web_custom/job.rb +11 -0
  72. data/lib/sidekiq/web_custom/processor.rb +82 -0
  73. data/lib/sidekiq/web_custom/queue.rb +12 -0
  74. data/lib/sidekiq/web_custom/timeout.rb +64 -0
  75. data/lib/sidekiq/web_custom/version.rb +14 -0
  76. data/lib/sidekiq/web_custom/web_action.rb +38 -0
  77. data/lib/sidekiq/web_custom/web_app.rb +46 -0
  78. data/sidekiq-web_custom.gemspec +37 -0
  79. metadata +151 -0
@@ -0,0 +1,97 @@
1
+ <header class="row">
2
+ <div class="col-sm-5">
3
+ <h3><%= t('ScheduledJobs') %></h3>
4
+ </div>
5
+ <% if @scheduled.size > 0 && @total_size > @count %>
6
+ <div class="col-sm-4">
7
+ <%= erb :_paging, locals: { url: "#{root_path}scheduled" } %>
8
+ </div>
9
+ <% end %>
10
+ <%= filtering('scheduled') %>
11
+ </header>
12
+
13
+ <% if @scheduled.size > 0 %>
14
+
15
+ <form action="<%= root_path %>scheduled" method="post">
16
+ <%= csrf_tag %>
17
+ <div class="table_container">
18
+ <table class="table table-striped table-bordered">
19
+ <thead>
20
+ <tr>
21
+ <th class="checkbox-column">
22
+ <input type="checkbox" class="check_all" />
23
+ </th>
24
+ <th><%= t('When') %></th>
25
+ <th><%= t('Queue') %></th>
26
+ <th><%= t('Job') %></th>
27
+ <th><%= t('Arguments') %></th>
28
+ <th><%= t('Actions') %></th>
29
+ </tr>
30
+ </thead>
31
+ <% @scheduled.each do |entry| %>
32
+ <tr>
33
+ <td>
34
+ <input type='checkbox' name='key[]' value='<%= job_params(entry.item, entry.score) %>' />
35
+ </td>
36
+ <td>
37
+ <a href="<%= root_path %>scheduled/<%= job_params(entry.item, entry.score) %>"><%= relative_time(entry.at) %></a>
38
+ </td>
39
+ <td>
40
+ <a href="<%= root_path %>queues/<%= entry.queue %>"><%= entry.queue %></a>
41
+ </td>
42
+ <td>
43
+ <%= entry.display_class %>
44
+ <%= display_tags(entry, "scheduled") %>
45
+ </td>
46
+ <td>
47
+ <div class="args"><%= display_args(entry.display_args) %></div>
48
+ </td>
49
+ <td>
50
+ __sidekiq_web_custom_replacement__
51
+ </td>
52
+ </tr>
53
+ <% end %>
54
+ </table>
55
+ </div>
56
+ <input class="btn btn-danger pull-right flip" type="submit" name="delete" value="<%= t('Delete') %>" />
57
+ <input class="btn btn-danger pull-right flip" type="submit" name="add_to_queue" value="<%= t('AddToQueue') %>" />
58
+ </form>
59
+ <% else %>
60
+ <div class="alert alert-success"><%= t('NoScheduledFound') %></div>
61
+ <% end %>
62
+
63
+ <div id="_sidekiq_web_block_screen_div_" style='display: none'>
64
+ <div id="_sidekiq_web_block_screen_text_"></div>
65
+ </div>
66
+
67
+ <script type="text/javascript">
68
+ function _sidekiq_web_block_screen(text) {
69
+ document.getElementById('_sidekiq_web_block_screen_text_').innerHTML = text
70
+ document.getElementById('_sidekiq_web_block_screen_div_').style.display = "block";
71
+ }
72
+ </script>
73
+
74
+ <style>
75
+ #_sidekiq_web_block_screen_div_ {
76
+ position: fixed;
77
+ display: none;
78
+ width: 100%;
79
+ height: 100%;
80
+ top: 0;
81
+ left: 0;
82
+ right: 0;
83
+ bottom: 0;
84
+ background-color: rgba(0,0,0,0.8);
85
+ z-index: 2;
86
+ }
87
+
88
+ #_sidekiq_web_block_screen_text_{
89
+ position: absolute;
90
+ top: 50%;
91
+ left: 50%;
92
+ font-size: 50px;
93
+ color: white;
94
+ transform: translate(-50%,-50%);
95
+ -ms-transform: translate(-50%,-50%);
96
+ }
97
+ </style>
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'byebug'
4
+ require_relative 'web_custom/version'
5
+
6
+ # require sidekiq/web first to ensure web_Action will prepend correctly
7
+ require 'sidekiq/web'
8
+ require 'sidekiq/web_custom/configuration'
9
+ require 'sidekiq/web_custom/web_action'
10
+ require 'sidekiq/web_custom/queue'
11
+ require 'sidekiq/web_custom/job'
12
+ require 'sidekiq/web_custom/web_app'
13
+
14
+ module Sidekiq
15
+ module WebCustom
16
+ class Error < StandardError; end
17
+ class ArgumentError < Error; end
18
+ class FileNotFound < Error; end
19
+ class StopExecution < Error; end
20
+ class ExecutionTimeExceeded < Error; end
21
+ class ConfigurationEstablished < Error; end
22
+
23
+ BREAK_BIT = '__sidekiq-web_custom-breakbit__'
24
+
25
+ def self.default_available_actions_mapping
26
+ @available_actions_mapping ||= begin
27
+ temp = {}
28
+ Dir["#{actions_root}/**/*.erb"].map do |erb_path|
29
+ base_path = File.basename(erb_path).split('.')[0]
30
+ second_half = erb_path.split(actions_root)[1]
31
+ action_type = second_half.split(base_path)[0]
32
+ action_type = action_type.delete('/').to_sym
33
+ temp[action_type] ||= {}
34
+ temp[action_type][base_path.to_sym] = erb_path
35
+ end
36
+ temp
37
+ end
38
+ end
39
+
40
+ def self.default_local_erb_mapping
41
+ @local_erb_mapping ||= Dir["#{local_erbs_root}/*.erb"].map do |erb_path|
42
+ [File.basename(erb_path).split('.')[0].to_sym, erb_path]
43
+ end.to_h
44
+ end
45
+
46
+ def self.actions_root
47
+ @actions_root ||= "#{local_erbs_root}/actions"
48
+ end
49
+
50
+ def self.local_erbs_root
51
+ @local_erbs_root ||= "#{root_path}/views"
52
+ end
53
+
54
+ def self.root_path
55
+ @root_path ||= File.dirname(__FILE__)
56
+ end
57
+
58
+ def self.config
59
+ @config ||= Configuration.new.tap do |t|
60
+ t.merge(base: :actions, params: default_available_actions_mapping)
61
+ t.merge(base: :local_erbs, params: default_local_erb_mapping)
62
+ end
63
+ end
64
+
65
+ def self.configure
66
+ yield config if block_given?
67
+
68
+ config.validate!
69
+ __inject_dependencies
70
+ end
71
+
72
+ def self.local_erb_mapping
73
+ config.local_erbs
74
+ end
75
+
76
+ def self.reset!
77
+ @config = nil
78
+ end
79
+
80
+ private
81
+
82
+ def self.__inject_dependencies
83
+ return if @__already_called
84
+
85
+ @__already_called = true
86
+ ::Sidekiq::WebAction.prepend WebAction
87
+ ::Sidekiq::Queue.prepend Queue
88
+ ::Sidekiq::Job.prepend Job
89
+ ::Sidekiq::Web.register WebApp
90
+ end
91
+ end
92
+ end
93
+
94
+ # dependent the error classes loaded on boot, requie after code is loaded
95
+ require 'sidekiq/web_custom/timeout'
@@ -0,0 +1,112 @@
1
+ module Sidekiq
2
+ module WebCustom
3
+ class Configuration
4
+
5
+ ALLOWED_BASED = [ACTIONS = :actions, LOCAL_ERBS = :local_erbs]
6
+ INTEGERS = [:drain_rate, :max_execution_time, :warn_execution_time]
7
+
8
+ DEFAULT_DRAIN_RATE = 10
9
+ DEFAULT_EXEC_TIME = 6
10
+ DEFAULT_WARN_TIME = 5
11
+
12
+ attr_reader *ALLOWED_BASED
13
+ attr_reader *INTEGERS
14
+
15
+ def initialize
16
+ ALLOWED_BASED.each do |var|
17
+ instance_variable_set(:"@#{var}", {})
18
+ end
19
+
20
+ @drain_rate = DEFAULT_DRAIN_RATE
21
+ @max_execution_time = DEFAULT_EXEC_TIME
22
+ @warn_execution_time = DEFAULT_WARN_TIME
23
+ end
24
+
25
+ INTEGERS.each do |int|
26
+ self.define_method("#{int}=") do |val|
27
+ unless _allow_write_block?
28
+ raise Sidekiq::WebCustom::ConfigurationEstablished, "Unable to assign [#{int}]. Assignment must happen on boot"
29
+ end
30
+
31
+ unless [Integer, Float].include?(val.class)
32
+ raise Sidekiq::WebCustom::ArgumentError, "Expected #{int} to be an integer or float"
33
+ end
34
+
35
+ instance_variable_set(:"@#{int}", val)
36
+ end
37
+ end
38
+
39
+ def merge(base:, params:, action_type: nil)
40
+ unless _allow_write_block?
41
+ raise Sidekiq::WebCustom::ConfigurationEstablished, "Unable to assign base [#{base}]. Assignment must happen on boot"
42
+ end
43
+ raise Sidekiq::WebCustom::ArgumentError, "Unexpected base: #{base}" unless ALLOWED_BASED.include?(base)
44
+ raise Sidekiq::WebCustom::ArgumentError, "Expected object for #{base} to be a Hash" unless params.is_a?(Hash)
45
+
46
+ value = instance_variable_get(:"@#{base}")
47
+ value =
48
+ if action_type && base == ACTIONS
49
+ value[action_type.to_sym] ||= {}
50
+ value[action_type.to_sym].merge!(params)
51
+ value
52
+ else
53
+ value.merge(params)
54
+ end
55
+ instance_variable_set(:"@#{base}", value)
56
+ end
57
+
58
+ def validate!
59
+ ALLOWED_BASED.each do |key|
60
+ value = instance_variable_get(:"@#{key}")
61
+
62
+ _validate!(value, key)
63
+ end
64
+
65
+ unless @warn_execution_time <= @max_execution_time
66
+ raise Sidekiq::WebCustom::ArgumentError, "Expected warn_execution_time to be less than max_execution_time"
67
+ end
68
+
69
+ unless actions.keys.all? { |k| local_erbs.keys.include?(k) }
70
+ raise Sidekiq::WebCustom::ArgumentError, "Unexpected actions keys#{actions.keys} -- Expected to be part of #{local_erbs.keys}"
71
+ end
72
+
73
+ define_convenienve_methods!
74
+ _unset_allow_write!
75
+ end
76
+
77
+ private
78
+
79
+ def _allow_write_block?
80
+ true
81
+ end
82
+
83
+ def _unset_allow_write!
84
+ define_singleton_method('_allow_write_block?') do
85
+ false
86
+ end
87
+ end
88
+
89
+ def define_convenienve_methods!
90
+ actions.keys.each do |key|
91
+ define_singleton_method("actions_for_#{key.to_s}") do
92
+ actions[key]
93
+ end
94
+ end
95
+ end
96
+
97
+ def _validate!(params, key, add: [])
98
+ params.each do |k, file|
99
+ return _validate!(file, k, add: add << key) if file.is_a? Hash
100
+ passed = [add, k].flatten.compact.join(':')
101
+
102
+
103
+ next if File.exist?(file)
104
+
105
+ raise Sidekiq::WebCustom::FileNotFound,
106
+ "#{key}.merge passed #{passed}: #{file}.\n" \
107
+ "The absolute file path does not exist."
108
+ end
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,11 @@
1
+ require 'sidekiq/web_custom/processor'
2
+
3
+ module Sidekiq
4
+ module WebCustom
5
+ module Job
6
+ def execute
7
+ Sidekiq::WebCustom::Processor.execute_job(job: self)
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,82 @@
1
+ require 'sidekiq/processor'
2
+
3
+ module Sidekiq
4
+ module WebCustom
5
+ class Processor < ::Sidekiq::Processor
6
+
7
+ def self.execute(max:, queue:, options: Sidekiq.options)
8
+ __processor__(queue: queue, options: options).__execute(max: max)
9
+ end
10
+
11
+ def self.execute_job(job:, options: Sidekiq.options)
12
+ __processor__(queue: job.queue, options: options).__execute_job(job: job)
13
+ rescue StandardError => _
14
+ false # error gets loggged downstream
15
+ end
16
+
17
+ def self.__processor__(queue:, options: Sidekiq.options)
18
+ options_temp = options.clone
19
+ queue = queue.is_a?(String) ? Sidekiq::Queue.new(queue) : queue
20
+
21
+ options_temp[:queues] = [queue.name]
22
+ klass = options_temp[:fetch]&.class || BasicFetch
23
+ options_temp[:fetch] = klass.new(options_temp)
24
+ new(manager: nil, options: options_temp, queue: queue)
25
+ end
26
+
27
+ def initialize(manager:, options:, queue:)
28
+ @__queue = queue
29
+ @__basic_fetch = options[:fetch].class == BasicFetch
30
+
31
+ super(manager, options)
32
+ end
33
+
34
+ def __execute_job(job:)
35
+ queue_name = "queue:#{job.queue}"
36
+ work_unit = Sidekiq::BasicFetch::UnitOfWork.new(queue_name, job.item.to_json)
37
+ begin
38
+ Sidekiq.logger.info "Manually processing individual work unit for #{work_unit.queue_name}"
39
+ process(work_unit)
40
+ rescue StandardError => e
41
+ Sidekiq.logger.error "Manually processed work unit failed with #{e.message}. Work unit will not be dequeued"
42
+ raise e
43
+ end
44
+
45
+ begin
46
+ job.delete
47
+ Sidekiq.logger.info { "Manually processed work unit sucessfully dequeued." }
48
+ rescue StandardError => e
49
+ Sidekiq.logger.fatal "Manually processed work unit failed to be dequeued. #{e.message}."
50
+ raise e
51
+ end
52
+
53
+ true
54
+ end
55
+
56
+ def __execute(max:)
57
+ count = 0
58
+ max.times do
59
+ break if @__queue.size <= 0
60
+
61
+ if Thread.current[Sidekiq::WebCustom::BREAK_BIT]
62
+ Sidekiq.logger.warn "Yikes -- Break bit has been set. Attempting to return in time. Completed #{count} of attempted #{max}"
63
+ break
64
+ end
65
+
66
+ Sidekiq.logger.info { "Manually processing next item in queue:[#{@__queue.name}]" }
67
+ process_one
68
+ count += 1
69
+
70
+ end
71
+ count
72
+ rescue Exception => ex
73
+ if @job && @__basic_fetch
74
+ Sidekiq.logger.fatal "Processor Execution interrupted. Lost Job #{@job.job}"
75
+ end
76
+ Sidekiq.logger.warn "Manual execution has terminated. Received error [#{ex.message}]"
77
+ return count
78
+ end
79
+ end
80
+ end
81
+ end
82
+
@@ -0,0 +1,12 @@
1
+ require 'sidekiq/web_custom/processor'
2
+
3
+ module Sidekiq
4
+ module WebCustom
5
+ module Queue
6
+ def drain(max:)
7
+ count = [size, max].min
8
+ Sidekiq::WebCustom::Processor.execute(max: count, queue: self)
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,64 @@
1
+ module Sidekiq
2
+ module WebCustom
3
+ module Timeout
4
+ module_function
5
+ DEFAULT_EXCEPTION = Sidekiq::WebCustom::ExecutionTimeExceeded
6
+ PROC = Proc.new do |warn_sec, timeout_sec, proc, exception, message, debug, &block|
7
+ puts "at: PROC; begining" if debug
8
+
9
+ sec_to_raise = timeout_sec - warn_sec
10
+ begin
11
+ from = debug ? "from #{caller_locations(1, 1)[0]}" : nil
12
+ x = Thread.current
13
+ # do everything in a new thread
14
+ y = Thread.start {
15
+ puts "at: PROC; second thread; about to start" if debug
16
+ Thread.current.name = from
17
+ # block for warning in new thread
18
+ begin
19
+ puts "at: PROC; second thread; starting warn time for #{warn_sec}'s " if debug
20
+ sleep warn_sec
21
+ rescue => e
22
+ x.raise e
23
+ else
24
+ # yield back during warning time so downstream can do some prep work
25
+ puts "at: PROC; second thread; trying to warn in main thread" if debug
26
+ proc.call(x, warn_sec)
27
+ end
28
+
29
+ # block additional seconds to raise for
30
+ begin
31
+ puts "at: PROC; second thread; starting violent exection time for #{sec_to_raise}'s " if debug
32
+ sleep sec_to_raise
33
+ rescue => e
34
+ puts "at: PROC; second thread; Error occured" if debug
35
+ x.raise e
36
+ else
37
+ puts "at: PROC; second thread; trying to raise in main thread" if debug
38
+ x.raise exception, message
39
+ end
40
+ }
41
+ puts "at: PROC; second thread; fully spooled" if debug
42
+ # after thread starts, yield back to calle function with max timout
43
+ block.call(timeout_sec)
44
+ ensure
45
+ if y
46
+ puts "at: PROC; Second thread still exists. Returned in time" if debug
47
+ y.kill
48
+ y.join
49
+ else
50
+ puts "at: PROC; Second thread no longer exists. Failed to return in time" if debug
51
+ end
52
+ end
53
+ end
54
+
55
+ def timeout(warn:, timeout:, proc: ->(_, _) {}, exception: DEFAULT_EXCEPTION, message: nil, debug: false, &block)
56
+ raise Sidekiq::WebCustom::ArgumentError, 'Block not given' unless block_given?
57
+
58
+ puts "at: timeout; valid bock given" if debug
59
+ message ||= "Execution exceeded #{timeout} seconds." if debug
60
+ PROC.call(warn, timeout, proc, exception, message, debug, &block)
61
+ end
62
+ end
63
+ end
64
+ end