fasten 0.5.4 → 0.6.0

data/lib/fasten/support/thread_worker.rb

@@ -0,0 +1,61 @@
+require 'fasten/std_thread_proxy'
+
+module Fasten
+  module Support
+    module ThreadWorker
+      attr_accessor :thread
+
+      def start
+        @queue = Queue.new
+
+        self.thread = Thread.new do
+          process_incoming_requests
+        end
+      end
+
+      def kill
+        log_info 'Removing worker'
+        thread.exit
+      rescue StandardError => error
+        log_warn "Ignoring error killing worker #{self}, error: #{error}"
+      ensure
+        @queue.clear
+      end
+
+      def send_request_to_child(task)
+        task.state = :RUNNING
+        task.worker = self
+        self.running_task = task
+        self.state = :RUNNING
+
+        @queue.push task
+      end
+
+      def receive_request_from_parent
+        @queue.pop
+      end
+
+      def send_response_to_parent(task)
+        log_info "Sending task response back to runner #{task}"
+
+        runner.queue.push task
+      end
+
+      def redirect_std(path)
+        StdThreadProxy.install
+
+        FileUtils.mkdir_p File.dirname(path)
+        @redirect_log = File.new path, 'a'
+        @redirect_log.sync = true
+        StdThreadProxy.thread_io = @redirect_log
+        logger.reopen(@redirect_log)
+      end
+
+      def restore_std
+        @redirect_log&.close
+        StdThreadProxy.thread_io = nil
+        logger.reopen(log_file)
+      end
+    end
+  end
+end

data/lib/fasten/support/ui.rb

@@ -0,0 +1,22 @@
+require 'fasten/ui/console'
+
+module Fasten
+  module Support
+    module UI
+      def ui
+        require 'fasten/ui/curses'
+        @ui ||= STDIN.tty? && STDOUT.tty? ? Fasten::UI::Curses.new(runner: self) : Fasten::UI::Console.new(runner: self)
+      rescue StandardError, LoadError
+        @ui = Fasten::UI::Console.new(runner: self)
+      end
+
+      def run_ui
+        ui.update
+
+        yield
+      ensure
+        ui.cleanup
+      end
+    end
+  end
+end

data/lib/fasten/support/yaml.rb

@@ -0,0 +1,54 @@
+require 'yaml'
+
+module Fasten
+  module Support
+    module Yaml
+      def load_yaml(path)
+        items = YAML.safe_load(File.read(path)).each do |name, params|
+          if params.is_a? String
+            params = { after: params }
+          elsif params.is_a? Hash
+            transform_params(params)
+          else
+            params = {}
+          end
+
+          add Fasten::Task.new({ name: name }.merge(params))
+        end
+
+        log_info "Loaded #{items.count} tasks from #{path}"
+      end
+
+      def save_yaml(path)
+        keys = %i[after shell]
+
+        items = task_list.map do |task|
+          data = task.to_h.select do |key, _val|
+            keys.include? key
+          end
+
+          [task.name, data]
+        end.to_h
+
+        File.write path, items.to_yaml
+
+        log_info "Loaded #{items.count} tasks into #{path}"
+      end
+
+      protected
+
+      def transform_params(params)
+        params.keys.each do |k|
+          val = params[k]
+
+          if val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
+            val = Regexp.new(match[1])
+          end
+
+          params[k.to_sym] = val
+          params.delete(k)
+        end
+      end
+    end
+  end
+end

data/lib/fasten/task.rb

@@ -1,25 +1,34 @@
+require 'fasten/support/state'
+
 module Fasten
   class Task
-    include Fasten::Logger
-    include Fasten::State
+    include Fasten::Support::State
 
     attr_accessor :name, :after, :shell, :ruby
-    attr_accessor :dependants, :depends, :request, :response, :worker, :run_score
+    attr_accessor :dependants, :depends, :request, :response, :worker, :run_score, :block
 
-    def initialize(name: nil, shell: nil, ruby: nil, request: nil, after: nil)
+    def initialize(name: nil, shell: nil, ruby: nil, block: nil, request: nil, after: nil)
       self.name = name
-      self.after = after
       self.shell = shell
       self.ruby = ruby
+      self.block = block
       self.request = request
+      self.after = after
     end
 
     def marshal_dump
-      [@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error]
+      [@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @block&.object_id, @error]
     end
 
     def marshal_load(data)
-      @name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error = data
+      @name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, block_id, @error = data
+      @block = ObjectSpace._id2ref block_id if block_id
+
+      raise "Sorry, unable to get block for task #{self}, please try using threads" if block_id && !@block.is_a?(Proc)
+    end
+
+    def kind
+      'task'
     end
 
     def to_s

data/lib/fasten/timeout_queue.rb

@@ -0,0 +1,36 @@
+module Fasten
+  class TimeoutQueue
+    def initialize
+      @mutex = Mutex.new
+      @queue = []
+      @received = ConditionVariable.new
+    end
+
+    def push(object)
+      @mutex.synchronize do
+        @queue << object
+        @received.signal
+      end
+    end
+
+    def receive_with_timeout(timeout = nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      @mutex.synchronize do
+        if timeout.nil?
+          # wait indefinitely until there is an element in the queue
+          @received.wait(@mutex) while @queue.empty?
+        elsif @queue.empty? && timeout != 0
+          # wait for element or timeout
+          timeout_time = timeout + Time.now.to_f
+          while @queue.empty? && (remaining_time = timeout_time - Time.now.to_f).positive?
+            @received.wait(@mutex, remaining_time)
+          end
+        end
+
+        items = []
+        items << @queue.shift until @queue.empty?
+
+        items
+      end
+    end
+  end
+end

data/lib/fasten/ui/console.rb

@@ -5,13 +5,13 @@ module Fasten
     class Console
       extend Forwardable
 
-      def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
-      def_delegators :executor, :name, :workers, :workers=, :state, :state=, :hformat
+      def_delegators :runner, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
+      def_delegators :runner, :name, :workers, :workers=, :state, :state=, :hformat
 
-      attr_accessor :executor
+      attr_accessor :runner
 
-      def initialize(executor:)
-        @executor = executor
+      def initialize(runner:)
+        @runner = runner
         @old = {
           task_done_list: [],
           task_error_list: []
@@ -22,7 +22,7 @@ module Fasten
         puts <<~FIN
 
           = == === ==== ===== ====== ======= ======== ========= ==========
-          Fasten your seatbelts! #{'💺' * workers}
+          Fasten your seatbelts! #{'💺' * workers} #{runner.use_threads ? 'threads' : 'processes'}
 
           #{name}
         FIN
@@ -50,7 +50,7 @@ module Fasten
         return unless old.count != orig.count
 
         (orig - old).each do |task|
-          puts "Time: #{hformat Time.new - executor.ini} #{message} #{hformat task.dif} Task #{task}"
+          puts "Time: #{hformat Time.new - runner.ini} #{message} #{hformat task.dif} Task #{task}"
           old << task
         end
       end

data/lib/fasten/ui/curses.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'curses'
 require 'forwardable'
 
 module Fasten
@@ -8,18 +9,18 @@ module Fasten
       include ::Curses
       extend Forwardable
 
-      def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
-      def_delegators :executor, :name, :workers, :workers=, :state, :state=
+      def_delegators :runner, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
+      def_delegators :runner, :name, :workers, :workers=, :state, :state=
 
-      attr_accessor :n_rows, :n_cols, :clear_needed, :message, :executor
+      attr_accessor :n_rows, :n_cols, :selected, :sel_index, :clear_needed, :message, :runner
 
       SPINNER_STR = '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
       SPINNER_LEN = SPINNER_STR.length
       PROGRESSBAR_STR = ' ▏▎▍▌▋▊▉'
       PROGRESSBAR_LEN = PROGRESSBAR_STR.length
 
-      def initialize(executor:)
-        @executor = executor
+      def initialize(runner:)
+        @runner = runner
       end
 
       def update
@@ -99,11 +100,17 @@ module Fasten
             self.message = "Can't remove 1 worker left, press [P] to pause"
           else
             self.workers -= 1
-            self.message = "Decreasing max workers to #{workers}"
+            self.message = "Decreasing workers to #{workers}"
           end
         elsif key == Curses::Key::RIGHT
           self.workers += 1
-          self.message = "Increasing max workers to #{workers}"
+          self.message = "Increasing workers to #{workers}"
+        elsif key == Curses::Key::DOWN
+          self.sel_index = sel_index ? [sel_index + 1, task_list.count - 1].min : 0
+          self.selected = task_list[sel_index]
+        elsif key == Curses::Key::UP
+          self.sel_index = sel_index ? [sel_index - 1, 0].max : task_list.count - 1
+          self.selected = task_list[sel_index]
         elsif key == 'q'
           self.message = 'Will quit when running tasks end'
           self.state = :QUITTING
@@ -122,7 +129,7 @@ module Fasten
         waiting_count = task_waiting_list.count
         workers_count = worker_list.count
 
-        "Procs: #{running_count} run #{workers_count - running_count} idle #{workers} max #{waiting_count} wait"
+        "Procs run: #{running_count} idle: #{workers_count - running_count} #{runner.use_threads ? 'threads' : 'processes'}: #{workers} wait: #{waiting_count}"
       end
 
       def ui_workers
@@ -140,13 +147,13 @@ module Fasten
       end
 
       def ui_state
-        if executor.running?
+        if runner.running?
           attrs = color_pair(2)
-        elsif executor.pausing?
+        elsif runner.pausing?
           attrs = color_pair(1) | A_BLINK | A_STANDOUT
-        elsif executor.paused?
+        elsif runner.paused?
           attrs = color_pair(1) | A_STANDOUT
-        elsif executor.quitting?
+        elsif runner.quitting?
           attrs = color_pair(3) | A_BLINK | A_STANDOUT
         end
 
@@ -188,19 +195,16 @@ module Fasten
       end
 
       def ui_task_color(task)
+        rev = task == selected ? A_REVERSE : 0
         case task.state
         when :RUNNING
-          color_pair(1) | A_TOP
+          color_pair(1) | A_TOP | rev
         when :FAIL
-          color_pair(3) | A_TOP
+          color_pair(3) | A_TOP | rev
         when :DONE
-          color_pair(2) | A_TOP
+          color_pair(2) | A_TOP | rev
         else
-          if task_waiting_list.include? task
-            A_TOP
-          else
-            color_pair(4) | A_DIM
-          end
+          task_waiting_list.include?(task) ? A_TOP | rev : color_pair(4) | A_DIM | rev
         end
       end
 
@@ -227,7 +231,7 @@ module Fasten
         count_done = task_done_list.count
         count_total = task_list.count
         tl = count_total.to_s.length
-        col_ini = ui_text_aligned(2, :left, format("Tasks: %#{tl}d/%d", count_done, count_total)) + 1
+        col_ini = ui_text_aligned(2, :left, format("Tasks %#{tl}d/%d", count_done, count_total)) + 1
         col_fin = n_cols - 5
         ui_text_aligned(2, :right, "#{(count_done * 100 / count_total).to_i}%") if count_total.positive?
 
@@ -253,7 +257,7 @@ module Fasten
             end
           else
             x = max + 1
-            last = executor.stats_last(task)
+            last = runner.stats_last(task)
             if task.dif
               str = format '  %.2f s', task.dif
             elsif last['avg'] && last['err']

data/lib/fasten/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fasten
-  VERSION = '0.5.4'
+  VERSION = '0.6.0'
 end

data/lib/fasten/worker.rb

@@ -1,3 +1,9 @@
+require 'English'
+require 'fasten/support/logger'
+require 'fasten/support/state'
+require 'fasten/support/fork_worker'
+require 'fasten/support/thread_worker'
+
 module Fasten
   class WorkerError < StandardError
     attr_reader :backtrace
@@ -9,97 +15,65 @@ module Fasten
   end
 
   class Worker
-    include Fasten::Logger
-    include Fasten::State
+    include Fasten::Support::Logger
+    include Fasten::Support::State
+
+    attr_accessor :runner, :name, :spinner, :child_read, :child_write, :parent_read, :parent_write, :running_task
 
-    attr_accessor :executor, :name, :spinner, :child_read, :child_write, :parent_read, :parent_write, :pid, :block, :running_task
+    def initialize(runner:, name: nil, use_threads: nil)
+      if use_threads
+        extend Fasten::Support::ThreadWorker
+      else
+        extend Fasten::Support::ForkWorker
+      end
 
-    def initialize(executor:, name: nil)
-      self.executor = executor
+      self.runner = runner
       self.name = name
       self.spinner = 0
-    end
 
-    def perform(task)
-      perform_shell(task) if task.shell
-      perform_ruby(task) if task.ruby
-      perform_block(task) if block
-    end
-
-    def perform_ruby(task)
-      task.response = eval task.ruby # rubocop:disable Security/Eval we trust our users ;-)
+      initialize_logger(log_file: runner&.log_file)
     end
 
-    def perform_shell(task)
-      result = system task.shell
-
-      raise "Command failed with exit code: #{$CHILD_STATUS.exitstatus}" unless result
-    end
-
-    def perform_block(task)
-      task.response = block.call(task.request)
-    end
-
-    def fork
-      create_pipes
-
-      self.pid = Process.fork do
-        close_parent_pipes
-
-        process_incoming_requests
+    def perform(task)
+      if task.ruby
+        perform_ruby(task)
+      elsif task.shell
+        perform_shell(task)
+      elsif task.block
+        perform_block(task, task.block)
       end
-
-      close_child_pipes
-    end
-
-    def send_request(task)
-      task.state = :RUNNING
-      task.worker = self
-      self.running_task = task
-      self.state = :RUNNING
-      Marshal.dump(task, parent_write)
     end
 
-    def receive_response
-      updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
-
-      %i[state ini fin dif response error].each { |key| running_task.send "#{key}=", updated_task.send(key) }
-
-      task = running_task
-      self.running_task = self.state = nil
-
-      task
+    def kind
+      'worker'
     end
 
-    def kill
-      log_info 'Removing worker'
-      Process.kill :KILL, pid
-      close_parent_pipes
-    rescue StandardError => error
-      log_warn "Ignoring error killing worker #{self}, error: #{error}"
+    def to_s
+      name
     end
 
     protected
 
-    def create_pipes
-      self.child_read, self.parent_write = IO.pipe
-      self.parent_read, self.child_write = IO.pipe
+    def perform_ruby(task)
+      task.response = eval task.ruby # rubocop:disable Security/Eval we trust our users ;-)
     end
 
-    def close_parent_pipes
-      parent_read.close unless parent_read.closed?
-      parent_write.close unless parent_write.closed?
+    def perform_shell(task)
+      shell_pid = spawn task.shell, out: @redirect_log, err: @redirect_log
+      Process.wait shell_pid
+      result = $CHILD_STATUS
+
+      raise "Command failed with exit code: #{result.exitstatus}" unless result.exitstatus.zero?
     end
 
-    def close_child_pipes
-      child_read.close unless child_read.closed?
-      child_write.close unless child_write.closed?
+    def perform_block(task, block)
+      task.response = instance_exec task.request, &block
     end
 
     def process_incoming_requests
       log_ini self, 'process_incoming_requests'
 
-      while (object = Marshal.load(child_read)) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
+      while (object = receive_request_from_parent)
         run_task(object) if object.is_a? Fasten::Task
       end
 
@@ -109,19 +83,18 @@ module Fasten
     end
 
     def run_task(task)
-      log_ini task, 'perform'
-      Fasten.logger.reopen(STDOUT)
-      redirect_std "#{executor.fasten_dir}/log/task/#{task.name}.log"
+      log_ini task, 'run_task'
+      redirect_std "#{runner.fasten_dir}/log/task/#{task.name}.log"
 
       perform_task task
-
+    ensure
       restore_std
-      Fasten.logger.reopen(executor.log_file)
-      log_fin task, 'perform'
+      logger.reopen(log_file)
+      log_fin task, 'run_task'
     end
 
     def perform_task(task)
-      log_ini task, 'perform'
+      log_ini task, 'perform_task'
 
       perform(task)
       task.state = :DONE
@@ -129,14 +102,8 @@ module Fasten
       task.state = :FAIL
       task.error = WorkerError.new(error)
     ensure
-      log_fin task, 'perform'
-      send_response(task)
-    end
-
-    def send_response(task)
-      log_info "Sending task response back to executor #{task}"
-      data = Marshal.dump(task)
-      child_write.write(data)
+      log_fin task, 'perform_task'
+      send_response_to_parent(task)
     end
   end
 end