fasten 0.5.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,61 @@
1
+ require 'fasten/std_thread_proxy'
2
+
3
+ module Fasten
4
+ module Support
5
+ module ThreadWorker
6
+ attr_accessor :thread
7
+
8
+ def start
9
+ @queue = Queue.new
10
+
11
+ self.thread = Thread.new do
12
+ process_incoming_requests
13
+ end
14
+ end
15
+
16
+ def kill
17
+ log_info 'Removing worker'
18
+ thread.exit
19
+ rescue StandardError => error
20
+ log_warn "Ignoring error killing worker #{self}, error: #{error}"
21
+ ensure
22
+ @queue.clear
23
+ end
24
+
25
+ def send_request_to_child(task)
26
+ task.state = :RUNNING
27
+ task.worker = self
28
+ self.running_task = task
29
+ self.state = :RUNNING
30
+
31
+ @queue.push task
32
+ end
33
+
34
+ def receive_request_from_parent
35
+ @queue.pop
36
+ end
37
+
38
+ def send_response_to_parent(task)
39
+ log_info "Sending task response back to runner #{task}"
40
+
41
+ runner.queue.push task
42
+ end
43
+
44
+ def redirect_std(path)
45
+ StdThreadProxy.install
46
+
47
+ FileUtils.mkdir_p File.dirname(path)
48
+ @redirect_log = File.new path, 'a'
49
+ @redirect_log.sync = true
50
+ StdThreadProxy.thread_io = @redirect_log
51
+ logger.reopen(@redirect_log)
52
+ end
53
+
54
+ def restore_std
55
+ @redirect_log&.close
56
+ StdThreadProxy.thread_io = nil
57
+ logger.reopen(log_file)
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,22 @@
1
+ require 'fasten/ui/console'
2
+
3
+ module Fasten
4
+ module Support
5
+ module UI
6
+ def ui
7
+ require 'fasten/ui/curses'
8
+ @ui ||= STDIN.tty? && STDOUT.tty? ? Fasten::UI::Curses.new(runner: self) : Fasten::UI::Console.new(runner: self)
9
+ rescue StandardError, LoadError
10
+ @ui = Fasten::UI::Console.new(runner: self)
11
+ end
12
+
13
+ def run_ui
14
+ ui.update
15
+
16
+ yield
17
+ ensure
18
+ ui.cleanup
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,54 @@
1
+ require 'yaml'
2
+
3
+ module Fasten
4
+ module Support
5
+ module Yaml
6
+ def load_yaml(path)
7
+ items = YAML.safe_load(File.read(path)).each do |name, params|
8
+ if params.is_a? String
9
+ params = { after: params }
10
+ elsif params.is_a? Hash
11
+ transform_params(params)
12
+ else
13
+ params = {}
14
+ end
15
+
16
+ add Fasten::Task.new({ name: name }.merge(params))
17
+ end
18
+
19
+ log_info "Loaded #{items.count} tasks from #{path}"
20
+ end
21
+
22
+ def save_yaml(path)
23
+ keys = %i[after shell]
24
+
25
+ items = task_list.map do |task|
26
+ data = task.to_h.select do |key, _val|
27
+ keys.include? key
28
+ end
29
+
30
+ [task.name, data]
31
+ end.to_h
32
+
33
+ File.write path, items.to_yaml
34
+
35
+ log_info "Loaded #{items.count} tasks into #{path}"
36
+ end
37
+
38
+ protected
39
+
40
+ def transform_params(params)
41
+ params.keys.each do |k|
42
+ val = params[k]
43
+
44
+ if val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
45
+ val = Regexp.new(match[1])
46
+ end
47
+
48
+ params[k.to_sym] = val
49
+ params.delete(k)
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -1,25 +1,34 @@
1
+ require 'fasten/support/state'
2
+
1
3
  module Fasten
2
4
  class Task
3
- include Fasten::Logger
4
- include Fasten::State
5
+ include Fasten::Support::State
5
6
 
6
7
  attr_accessor :name, :after, :shell, :ruby
7
- attr_accessor :dependants, :depends, :request, :response, :worker, :run_score
8
+ attr_accessor :dependants, :depends, :request, :response, :worker, :run_score, :block
8
9
 
9
- def initialize(name: nil, shell: nil, ruby: nil, request: nil, after: nil)
10
+ def initialize(name: nil, shell: nil, ruby: nil, block: nil, request: nil, after: nil)
10
11
  self.name = name
11
- self.after = after
12
12
  self.shell = shell
13
13
  self.ruby = ruby
14
+ self.block = block
14
15
  self.request = request
16
+ self.after = after
15
17
  end
16
18
 
17
19
  def marshal_dump
18
- [@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error]
20
+ [@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @block&.object_id, @error]
19
21
  end
20
22
 
21
23
  def marshal_load(data)
22
- @name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error = data
24
+ @name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, block_id, @error = data
25
+ @block = ObjectSpace._id2ref block_id if block_id
26
+
27
+ raise "Sorry, unable to get block for task #{self}, please try using threads" if block_id && !@block.is_a?(Proc)
28
+ end
29
+
30
+ def kind
31
+ 'task'
23
32
  end
24
33
 
25
34
  def to_s
@@ -0,0 +1,36 @@
1
+ module Fasten
2
+ class TimeoutQueue
3
+ def initialize
4
+ @mutex = Mutex.new
5
+ @queue = []
6
+ @received = ConditionVariable.new
7
+ end
8
+
9
+ def push(object)
10
+ @mutex.synchronize do
11
+ @queue << object
12
+ @received.signal
13
+ end
14
+ end
15
+
16
+ def receive_with_timeout(timeout = nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
17
+ @mutex.synchronize do
18
+ if timeout.nil?
19
+ # wait indefinitely until there is an element in the queue
20
+ @received.wait(@mutex) while @queue.empty?
21
+ elsif @queue.empty? && timeout != 0
22
+ # wait for element or timeout
23
+ timeout_time = timeout + Time.now.to_f
24
+ while @queue.empty? && (remaining_time = timeout_time - Time.now.to_f).positive?
25
+ @received.wait(@mutex, remaining_time)
26
+ end
27
+ end
28
+
29
+ items = []
30
+ items << @queue.shift until @queue.empty?
31
+
32
+ items
33
+ end
34
+ end
35
+ end
36
+ end
@@ -5,13 +5,13 @@ module Fasten
5
5
  class Console
6
6
  extend Forwardable
7
7
 
8
- def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
9
- def_delegators :executor, :name, :workers, :workers=, :state, :state=, :hformat
8
+ def_delegators :runner, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
9
+ def_delegators :runner, :name, :workers, :workers=, :state, :state=, :hformat
10
10
 
11
- attr_accessor :executor
11
+ attr_accessor :runner
12
12
 
13
- def initialize(executor:)
14
- @executor = executor
13
+ def initialize(runner:)
14
+ @runner = runner
15
15
  @old = {
16
16
  task_done_list: [],
17
17
  task_error_list: []
@@ -22,7 +22,7 @@ module Fasten
22
22
  puts <<~FIN
23
23
 
24
24
  = == === ==== ===== ====== ======= ======== ========= ==========
25
- Fasten your seatbelts! #{'💺' * workers}
25
+ Fasten your seatbelts! #{'💺' * workers} #{runner.use_threads ? 'threads' : 'processes'}
26
26
 
27
27
  #{name}
28
28
  FIN
@@ -50,7 +50,7 @@ module Fasten
50
50
  return unless old.count != orig.count
51
51
 
52
52
  (orig - old).each do |task|
53
- puts "Time: #{hformat Time.new - executor.ini} #{message} #{hformat task.dif} Task #{task}"
53
+ puts "Time: #{hformat Time.new - runner.ini} #{message} #{hformat task.dif} Task #{task}"
54
54
  old << task
55
55
  end
56
56
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'curses'
3
4
  require 'forwardable'
4
5
 
5
6
  module Fasten
@@ -8,18 +9,18 @@ module Fasten
8
9
  include ::Curses
9
10
  extend Forwardable
10
11
 
11
- def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
12
- def_delegators :executor, :name, :workers, :workers=, :state, :state=
12
+ def_delegators :runner, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
13
+ def_delegators :runner, :name, :workers, :workers=, :state, :state=
13
14
 
14
- attr_accessor :n_rows, :n_cols, :clear_needed, :message, :executor
15
+ attr_accessor :n_rows, :n_cols, :selected, :sel_index, :clear_needed, :message, :runner
15
16
 
16
17
  SPINNER_STR = '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
17
18
  SPINNER_LEN = SPINNER_STR.length
18
19
  PROGRESSBAR_STR = ' ▏▎▍▌▋▊▉'
19
20
  PROGRESSBAR_LEN = PROGRESSBAR_STR.length
20
21
 
21
- def initialize(executor:)
22
- @executor = executor
22
+ def initialize(runner:)
23
+ @runner = runner
23
24
  end
24
25
 
25
26
  def update
@@ -99,11 +100,17 @@ module Fasten
99
100
  self.message = "Can't remove 1 worker left, press [P] to pause"
100
101
  else
101
102
  self.workers -= 1
102
- self.message = "Decreasing max workers to #{workers}"
103
+ self.message = "Decreasing workers to #{workers}"
103
104
  end
104
105
  elsif key == Curses::Key::RIGHT
105
106
  self.workers += 1
106
- self.message = "Increasing max workers to #{workers}"
107
+ self.message = "Increasing workers to #{workers}"
108
+ elsif key == Curses::Key::DOWN
109
+ self.sel_index = sel_index ? [sel_index + 1, task_list.count - 1].min : 0
110
+ self.selected = task_list[sel_index]
111
+ elsif key == Curses::Key::UP
112
+ self.sel_index = sel_index ? [sel_index - 1, 0].max : task_list.count - 1
113
+ self.selected = task_list[sel_index]
107
114
  elsif key == 'q'
108
115
  self.message = 'Will quit when running tasks end'
109
116
  self.state = :QUITTING
@@ -122,7 +129,7 @@ module Fasten
122
129
  waiting_count = task_waiting_list.count
123
130
  workers_count = worker_list.count
124
131
 
125
- "Procs: #{running_count} run #{workers_count - running_count} idle #{workers} max #{waiting_count} wait"
132
+ "Procs run: #{running_count} idle: #{workers_count - running_count} #{runner.use_threads ? 'threads' : 'processes'}: #{workers} wait: #{waiting_count}"
126
133
  end
127
134
 
128
135
  def ui_workers
@@ -140,13 +147,13 @@ module Fasten
140
147
  end
141
148
 
142
149
  def ui_state
143
- if executor.running?
150
+ if runner.running?
144
151
  attrs = color_pair(2)
145
- elsif executor.pausing?
152
+ elsif runner.pausing?
146
153
  attrs = color_pair(1) | A_BLINK | A_STANDOUT
147
- elsif executor.paused?
154
+ elsif runner.paused?
148
155
  attrs = color_pair(1) | A_STANDOUT
149
- elsif executor.quitting?
156
+ elsif runner.quitting?
150
157
  attrs = color_pair(3) | A_BLINK | A_STANDOUT
151
158
  end
152
159
 
@@ -188,19 +195,16 @@ module Fasten
188
195
  end
189
196
 
190
197
  def ui_task_color(task)
198
+ rev = task == selected ? A_REVERSE : 0
191
199
  case task.state
192
200
  when :RUNNING
193
- color_pair(1) | A_TOP
201
+ color_pair(1) | A_TOP | rev
194
202
  when :FAIL
195
- color_pair(3) | A_TOP
203
+ color_pair(3) | A_TOP | rev
196
204
  when :DONE
197
- color_pair(2) | A_TOP
205
+ color_pair(2) | A_TOP | rev
198
206
  else
199
- if task_waiting_list.include? task
200
- A_TOP
201
- else
202
- color_pair(4) | A_DIM
203
- end
207
+ task_waiting_list.include?(task) ? A_TOP | rev : color_pair(4) | A_DIM | rev
204
208
  end
205
209
  end
206
210
 
@@ -227,7 +231,7 @@ module Fasten
227
231
  count_done = task_done_list.count
228
232
  count_total = task_list.count
229
233
  tl = count_total.to_s.length
230
- col_ini = ui_text_aligned(2, :left, format("Tasks: %#{tl}d/%d", count_done, count_total)) + 1
234
+ col_ini = ui_text_aligned(2, :left, format("Tasks %#{tl}d/%d", count_done, count_total)) + 1
231
235
  col_fin = n_cols - 5
232
236
  ui_text_aligned(2, :right, "#{(count_done * 100 / count_total).to_i}%") if count_total.positive?
233
237
 
@@ -253,7 +257,7 @@ module Fasten
253
257
  end
254
258
  else
255
259
  x = max + 1
256
- last = executor.stats_last(task)
260
+ last = runner.stats_last(task)
257
261
  if task.dif
258
262
  str = format ' %.2f s', task.dif
259
263
  elsif last['avg'] && last['err']
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Fasten
4
- VERSION = '0.5.4'
4
+ VERSION = '0.6.0'
5
5
  end
@@ -1,3 +1,9 @@
1
+ require 'English'
2
+ require 'fasten/support/logger'
3
+ require 'fasten/support/state'
4
+ require 'fasten/support/fork_worker'
5
+ require 'fasten/support/thread_worker'
6
+
1
7
  module Fasten
2
8
  class WorkerError < StandardError
3
9
  attr_reader :backtrace
@@ -9,97 +15,65 @@ module Fasten
9
15
  end
10
16
 
11
17
  class Worker
12
- include Fasten::Logger
13
- include Fasten::State
18
+ include Fasten::Support::Logger
19
+ include Fasten::Support::State
20
+
21
+ attr_accessor :runner, :name, :spinner, :child_read, :child_write, :parent_read, :parent_write, :running_task
14
22
 
15
- attr_accessor :executor, :name, :spinner, :child_read, :child_write, :parent_read, :parent_write, :pid, :block, :running_task
23
+ def initialize(runner:, name: nil, use_threads: nil)
24
+ if use_threads
25
+ extend Fasten::Support::ThreadWorker
26
+ else
27
+ extend Fasten::Support::ForkWorker
28
+ end
16
29
 
17
- def initialize(executor:, name: nil)
18
- self.executor = executor
30
+ self.runner = runner
19
31
  self.name = name
20
32
  self.spinner = 0
21
- end
22
33
 
23
- def perform(task)
24
- perform_shell(task) if task.shell
25
- perform_ruby(task) if task.ruby
26
- perform_block(task) if block
27
- end
28
-
29
- def perform_ruby(task)
30
- task.response = eval task.ruby # rubocop:disable Security/Eval we trust our users ;-)
34
+ initialize_logger(log_file: runner&.log_file)
31
35
  end
32
36
 
33
- def perform_shell(task)
34
- result = system task.shell
35
-
36
- raise "Command failed with exit code: #{$CHILD_STATUS.exitstatus}" unless result
37
- end
38
-
39
- def perform_block(task)
40
- task.response = block.call(task.request)
41
- end
42
-
43
- def fork
44
- create_pipes
45
-
46
- self.pid = Process.fork do
47
- close_parent_pipes
48
-
49
- process_incoming_requests
37
+ def perform(task)
38
+ if task.ruby
39
+ perform_ruby(task)
40
+ elsif task.shell
41
+ perform_shell(task)
42
+ elsif task.block
43
+ perform_block(task, task.block)
50
44
  end
51
-
52
- close_child_pipes
53
- end
54
-
55
- def send_request(task)
56
- task.state = :RUNNING
57
- task.worker = self
58
- self.running_task = task
59
- self.state = :RUNNING
60
- Marshal.dump(task, parent_write)
61
45
  end
62
46
 
63
- def receive_response
64
- updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
65
-
66
- %i[state ini fin dif response error].each { |key| running_task.send "#{key}=", updated_task.send(key) }
67
-
68
- task = running_task
69
- self.running_task = self.state = nil
70
-
71
- task
47
+ def kind
48
+ 'worker'
72
49
  end
73
50
 
74
- def kill
75
- log_info 'Removing worker'
76
- Process.kill :KILL, pid
77
- close_parent_pipes
78
- rescue StandardError => error
79
- log_warn "Ignoring error killing worker #{self}, error: #{error}"
51
+ def to_s
52
+ name
80
53
  end
81
54
 
82
55
  protected
83
56
 
84
- def create_pipes
85
- self.child_read, self.parent_write = IO.pipe
86
- self.parent_read, self.child_write = IO.pipe
57
+ def perform_ruby(task)
58
+ task.response = eval task.ruby # rubocop:disable Security/Eval we trust our users ;-)
87
59
  end
88
60
 
89
- def close_parent_pipes
90
- parent_read.close unless parent_read.closed?
91
- parent_write.close unless parent_write.closed?
61
+ def perform_shell(task)
62
+ shell_pid = spawn task.shell, out: @redirect_log, err: @redirect_log
63
+ Process.wait shell_pid
64
+ result = $CHILD_STATUS
65
+
66
+ raise "Command failed with exit code: #{result.exitstatus}" unless result.exitstatus.zero?
92
67
  end
93
68
 
94
- def close_child_pipes
95
- child_read.close unless child_read.closed?
96
- child_write.close unless child_write.closed?
69
+ def perform_block(task, block)
70
+ task.response = instance_exec task.request, &block
97
71
  end
98
72
 
99
73
  def process_incoming_requests
100
74
  log_ini self, 'process_incoming_requests'
101
75
 
102
- while (object = Marshal.load(child_read)) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
76
+ while (object = receive_request_from_parent)
103
77
  run_task(object) if object.is_a? Fasten::Task
104
78
  end
105
79
 
@@ -109,19 +83,18 @@ module Fasten
109
83
  end
110
84
 
111
85
  def run_task(task)
112
- log_ini task, 'perform'
113
- Fasten.logger.reopen(STDOUT)
114
- redirect_std "#{executor.fasten_dir}/log/task/#{task.name}.log"
86
+ log_ini task, 'run_task'
87
+ redirect_std "#{runner.fasten_dir}/log/task/#{task.name}.log"
115
88
 
116
89
  perform_task task
117
-
90
+ ensure
118
91
  restore_std
119
- Fasten.logger.reopen(executor.log_file)
120
- log_fin task, 'perform'
92
+ logger.reopen(log_file)
93
+ log_fin task, 'run_task'
121
94
  end
122
95
 
123
96
  def perform_task(task)
124
- log_ini task, 'perform'
97
+ log_ini task, 'perform_task'
125
98
 
126
99
  perform(task)
127
100
  task.state = :DONE
@@ -129,14 +102,8 @@ module Fasten
129
102
  task.state = :FAIL
130
103
  task.error = WorkerError.new(error)
131
104
  ensure
132
- log_fin task, 'perform'
133
- send_response(task)
134
- end
135
-
136
- def send_response(task)
137
- log_info "Sending task response back to executor #{task}"
138
- data = Marshal.dump(task)
139
- child_write.write(data)
105
+ log_fin task, 'perform_task'
106
+ send_response_to_parent(task)
140
107
  end
141
108
  end
142
109
  end