fasten 0.5.2 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d8a5e3862d3e3572c5fd0316458efd9676999cf764b15109870fc2886f3e4e4
-  data.tar.gz: 9e8b633cac41ad9d10b08d615f63c1010abe7f375579620e52ce90456f0b53cb
+  metadata.gz: e475b26b22ab143005ca207f29ee7036571b71934a7fa2bba0ea2be74ba1aaab
+  data.tar.gz: 39429fea342cf0c43e1b84cef14e897a1d12f56a5f11e4c555c28176bfa1cdda
 SHA512:
-  metadata.gz: 743f18548d13b69747d11057c4bd8923448642cdaafaa8f91d68fa5156d613db15094d2194b2d16a1fc2a88493cc52741b76d1e652b3db9cf4a2d9ec3991d332
-  data.tar.gz: 5a1f5d9d57b26b31ff90867a999ea285cef8e09e4972e648baffe7887d732680f64d0aea09d19ef728616b9d430ae89142e448d49c9e7a979e8e22bc1fb50c76
+  metadata.gz: 7bd2923a0ae1d4cf5f31b36a3999be1bfec058a99acdd381712e91454e2ed9baf7193bb87de835359b300a9317cdd47f2c92e850c54a36e7dcf26dd88776d720
+  data.tar.gz: b4307bc7ee694ad3740b0c0b361d4c5a4888f5e405d32b6aa91ceee8c0023ee1a6421771a6915a2f3916d0610cdf5136f329f47076e33bb066e7e3eb4fb572e3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fasten (0.5.2)
+    fasten (0.5.4)
       binding_of_caller
       curses
       hirb

data/lib/fasten/dag.rb

@@ -45,7 +45,7 @@ module Fasten
     end
 
     def next_task
-      task_waiting_list.sort_by!(&:run_score).pop
+      task_waiting_list.shift
     end
 
     def task_waiting_list
@@ -67,16 +67,30 @@ module Fasten
       @task_waiting_list ||= []
       @task_pending_list -= move_list
       @task_waiting_list += move_list
+      @task_waiting_list.sort_by!.with_index do |x, index|
+        x.state = :WAIT
+        [-x.run_score, index]
+      end
     end
 
     def reset_tasks
       @task_pending_list.clear
       @task_done_list.clear
+      @task_error_list.clear
+
       @task_list.each do |task|
         task.dependants = []
         task.depends = []
         task.run_score = 0
-        task.done ? @task_done_list << task : @task_pending_list << task
+
+        if task.state == :DONE
+          @task_done_list << task
+        elsif task.state == :FAIL
+          @task_error_list << task
+        else
+          task.state = :IDLE
+          @task_pending_list << task
+        end
       end
     end
 

data/lib/fasten/executor.rb

@@ -1,21 +1,28 @@
 module Fasten
-  class Executor < Task
+  class Executor
     include Fasten::Logger
+    include Fasten::State
     include Fasten::DAG
     include Fasten::UI
-    include Fasten::LoadSave
+    include Fasten::Yaml
     include Fasten::Stats
 
-    def initialize(name: nil, developer: STDIN.tty? && STDOUT.tty?, workers: Parallel.physical_processor_count, worker_class: Fasten::Worker, fasten_dir: '.fasten')
-      setup_stats(name)
-      super name: name || "#{self.class} #{$PID}", workers: workers, pid: $PID, state: :IDLE, worker_class: worker_class, fasten_dir: fasten_dir, developer: developer
+    attr_accessor :name, :workers, :worker_class, :pid, :fasten_dir, :developer, :stats, :worker_list, :block
+
+    def initialize(name: nil, developer: STDIN.tty? && STDOUT.tty?, workers: Parallel.physical_processor_count, worker_class: Worker, fasten_dir: '.fasten')
+      self.stats = name && true
+      self.name = name || "#{self.class} #{$PID}"
+      self.state = :IDLE
+      self.workers = workers
+      self.worker_class = worker_class
+      self.fasten_dir = fasten_dir
+      self.developer = developer
+
       initialize_dag
+      initialize_stats
+      initialize_logger
 
       self.worker_list = []
-      log_path = "#{fasten_dir}/log/executor/#{self.name}.log"
-      FileUtils.mkdir_p File.dirname(log_path)
-      self.log_file = File.new(log_path, 'a')
-      Fasten.logger.reopen log_file
     end
 
     def perform
@@ -62,7 +69,7 @@ module Fasten
     def check_state
       if state == :PAUSING && no_running_tasks?
         self.state = :PAUSED
-        self.ui.message = nil
+        ui.message = nil
         ui.force_clear
       elsif state == :QUITTING && no_running_tasks?
         self.state = :QUIT
@@ -101,17 +108,22 @@ module Fasten
       end
     end
 
-    def raise_error_in_failure
-      return unless tasks_failed?
-
+    def show_error_tasks
       task_error_list.each do |task|
         log_info "task: #{task} error:#{task.error}\n#{task.error&.backtrace&.join("\n")}"
       end
+    end
+
+    def raise_error_in_failure
+      return unless tasks_failed?
+
+      show_error_tasks
+
+      message = "Stopping because the following tasks failed: #{task_error_list.map(&:to_s).join(', ')}"
 
       if developer
         ui.cleanup
-        puts "Stopping because the following tasks failed:\n"
-        task_error_list.map(&:to_s).each { |x| puts "  #{x}" }
+        puts message
 
         puts 'Entering development console'
 
@@ -119,7 +131,7 @@ module Fasten
       else
         remove_all_workers
 
-        raise "Stopping because the following tasks failed: #{task_error_list.map(&:to_s).join(', ')}"
+        raise message
       end
     end
 

data/lib/fasten/logger.rb

@@ -4,6 +4,8 @@ module Fasten
   end
 
   module Logger
+    attr_accessor :log_file
+
     %w[debug info error].each do |method|
       define_method "log_#{method}" do |msg|
         return unless Fasten.logger.respond_to?(method)
@@ -14,16 +16,23 @@ module Fasten
       end
     end
 
+    def initialize_logger
+      log_path = "#{fasten_dir}/log/executor/#{name}.log"
+      FileUtils.mkdir_p File.dirname(log_path)
+      self.log_file = File.new(log_path, 'a')
+      Fasten.logger.reopen log_file
+    end
+
     def log_ini(object, message = nil)
       object.ini ||= Time.new
-      log_info "Init #{object.class} #{object} #{message}"
+      log_info "Ini #{object.state} #{object.class} #{object} #{message}"
     end
 
     def log_fin(object, message = nil)
       object.fin ||= Time.new
       object.dif = object.fin - object.ini
 
-      log_info "Done #{object.class} #{object} #{message} in #{object.dif}"
+      log_info "Fin #{object.state} #{object.class} #{object} #{message} in #{object.dif}"
     end
 
     def redirect_std(path)

data/lib/fasten/state.rb

@@ -0,0 +1,30 @@
+module Fasten
+  module State
+    attr_accessor :error, :ini, :fin, :dif, :last
+    attr_writer :state
+
+    def state
+      @state || :IDLE
+    end
+
+    def running?
+      state == :RUNNING
+    end
+
+    def idle?
+      state == :IDLE
+    end
+
+    def pausing?
+      state == :PAUSING
+    end
+
+    def paused?
+      state == :PAUSED
+    end
+
+    def quitting?
+      state == :QUITTING
+    end
+  end
+end

data/lib/fasten/stats.rb

@@ -1,30 +1,78 @@
 module Fasten
   module Stats
+    attr_writer :stats_data, :stats_entries
+    attr_reader :stats_path
+
+    def initialize_stats
+      return unless stats
+
+      @stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME']
+      FileUtils.mkdir_p File.dirname(@stats_path)
+    rescue StandardError
+      @stats_path = nil
+    end
+
+    def load_stats
+      return unless @stats_path && File.exist?(@stats_path)
+
+      self.stats_data = []
+      CSV.foreach(@stats_path, headers: true) do |row|
+        stats_data << row.to_h
+      end
+
+      @task_waiting_list = nil
+    rescue StandardError
+      nil
+    ensure
+      self.stats ||= {}
+    end
+
+    def save_stats
+      return unless @stats_path && stats_data
+
+      keys = %w[state kind name run cnt avg std err]
+
+      CSV.open(@stats_path, 'wb') do |csv|
+        csv << keys
+
+        stats_data.each do |data|
+          csv << keys.map { |i| data[i] }
+        end
+      end
+    end
+
     def stats_create_entry(state, target)
-      {
-        'state' => state.to_s,
-        'kind' => target.is_a?(Fasten::Executor) ? 'executor' : 'task',
-        'name' => target.name,
-        'ini' => target.ini.to_f,
-        'fin' => target.fin.to_f,
-        'run' => target.fin - target.ini
-      }
+      { 'state' => state.to_s,
+        'kind'  => stats_kind_for(target),
+        'name'  => target.name,
+        'ini'   => target.ini.to_f,
+        'fin'   => target.fin.to_f,
+        'run'   => target.fin - target.ini,
+        'worker' => target.respond_to?(:worker) ? target.worker.name : nil }
+    end
+
+    def stats_data
+      @stats_data ||= []
+    end
+
+    def stats_entries
+      @stats_entries ||= []
+    end
+
+    def stats_kind_for(object)
+      object.is_a?(Fasten::Executor) ? 'executor' : 'task'
     end
 
     def stats_add_entry(state, target)
       return unless target.ini && target.fin
 
       entry = stats_create_entry(state, target)
-      self.stats_data ||= []
-      self.stats_entries ||= []
       stats_data << entry
       stats_entries << entry
 
       history = stats_history(entry)
 
-      update_cnt(history, entry)
-      update_avg(history, entry)
-      update_std(history, entry)
+      update_stats(history, entry)
     end
 
     FLOAT_FORMATTER = ->(f) { format('%7.3f', f) }
@@ -61,8 +109,8 @@ module Fasten
       sub, tot = stats_table_run
 
       Hirb::Console.render_output(stats_entries,
-                                  fields: %w[state kind name run cnt avg std], unicode: true, class: 'Hirb::Helpers::AutoTable',
-                                  filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER },
+                                  fields: %w[state kind name run cnt avg std err worker], unicode: true, class: 'Hirb::Helpers::AutoTable',
+                                  filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER, 'err' => FLOAT_FORMATTER },
                                   description: false)
 
       puts format('∑tasks: %<task>s ∑executed: %<executed>s saved: %<saved>s workers: %<workers>s',
@@ -73,16 +121,17 @@ module Fasten
       stats_data.select { |e| e['state'] == entry['state'] && e['kind'] == entry['kind'] && e['name'] == entry['name'] }
     end
 
-    def update_cnt(history, entry)
-      entry['cnt'] = history.size
-    end
+    def stats_last(item)
+      return item.last if item.last
 
-    def update_avg(history, entry)
-      entry['avg'] = history.inject(0.0) { |s, x| s + x['run'].to_f } / history.size
+      item.last = stats_data.select { |e| e['kind'] == stats_kind_for(item) && e['name'] == item.name }.last || {}
     end
 
-    def update_std(history, entry)
-      entry['std'] = Math.sqrt(history.inject(0.0) { |v, x| v + (x['run'].to_f - entry['avg'])**2 })
+    def update_stats(history, entry)
+      entry['cnt'] = count = history.size
+      entry['avg'] = avg = history.inject(0.0) { |s, x| s + x['run'].to_f } / count
+      entry['std'] = std = Math.sqrt(history.inject(0.0) { |v, x| v + (x['run'].to_f - avg)**2 })
+      entry['err'] = std / Math.sqrt(count) if count.positive?
     end
   end
 end

data/lib/fasten/task.rb

@@ -1,6 +1,26 @@
 module Fasten
-  class Task < OpenStruct
+  class Task
     include Fasten::Logger
+    include Fasten::State
+
+    attr_accessor :name, :after, :shell, :ruby
+    attr_accessor :dependants, :depends, :request, :response, :worker, :run_score
+
+    def initialize(name: nil, shell: nil, ruby: nil, request: nil, after: nil)
+      self.name = name
+      self.after = after
+      self.shell = shell
+      self.ruby = ruby
+      self.request = request
+    end
+
+    def marshal_dump
+      [@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error]
+    end
+
+    def marshal_load(data)
+      @name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error = data
+    end
 
     def to_s
       name

data/lib/fasten/ui/console.rb

@@ -4,6 +4,7 @@ module Fasten
   module UI
     class Console
       extend Forwardable
+
       def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
       def_delegators :executor, :name, :workers, :workers=, :state, :state=, :hformat
 

data/lib/fasten/ui/curses.rb

@@ -7,6 +7,7 @@ module Fasten
     class Curses
       include ::Curses
       extend Forwardable
+
       def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
       def_delegators :executor, :name, :workers, :workers=, :state, :state=
 
@@ -139,13 +140,13 @@ module Fasten
       end
 
       def ui_state
-        if state == :RUNNING
+        if executor.running?
           attrs = color_pair(2)
-        elsif state == :PAUSING
+        elsif executor.pausing?
           attrs = color_pair(1) | A_BLINK | A_STANDOUT
-        elsif state == :PAUSED
+        elsif executor.paused?
           attrs = color_pair(1) | A_STANDOUT
-        elsif state == :QUITTING
+        elsif executor.quitting?
           attrs = color_pair(3) | A_BLINK | A_STANDOUT
         end
 
@@ -161,7 +162,7 @@ module Fasten
         col_ini.upto col_fin do |col|
           setpos row, col
           count -= slice
-          if count.positive?
+          if count >= 0
             addstr PROGRESSBAR_STR[-1]
           elsif count > -slice
             addstr PROGRESSBAR_STR[(count * PROGRESSBAR_LEN / slice) % PROGRESSBAR_LEN]
@@ -179,8 +180,10 @@ module Fasten
           '✘'
         when :DONE
           '✔'
-        else
+        when :WAIT
           '…'
+        else
+          ' '
         end
       end
 
@@ -193,7 +196,11 @@ module Fasten
         when :DONE
           color_pair(2) | A_TOP
         else
-          color_pair(4) | A_TOP
+          if task_waiting_list.include? task
+            A_TOP
+          else
+            color_pair(4) | A_DIM
+          end
         end
       end
 
@@ -227,7 +234,7 @@ module Fasten
         ui_progressbar(2, col_ini, col_fin, count_done, count_total)
 
         max = 2
-        list = task_list.sort_by(&:run_score)
+        list = task_list.sort_by.with_index { |x, index| [x.run_score, index] }
         list.each_with_index do |task, index|
           next if 3 + index >= n_rows
 
@@ -238,16 +245,23 @@ module Fasten
         list.each_with_index do |task, index|
           next if 3 + index >= n_rows
 
-          if task.dif
-            setpos 3 + index, max + 2
-            ui_task_string(task, 3 + index, max + 2, str: format('%.2f s', task.dif))
-          elsif task.depends && !task.depends.empty?
-            setpos 3 + index, max
-            x = max + 2
-            addstr ':'
+          if task.depends && !task.depends.empty?
+            x = max
+            x = ui_task_string(task, 3 + index, x, str: ':') + 1
             task.depends.each do |dependant_task|
               x = ui_task_string(dependant_task, 3 + index, x) + 1
             end
+          else
+            x = max + 1
+            last = executor.stats_last(task)
+            if task.dif
+              str = format '  %.2f s', task.dif
+            elsif last['avg'] && last['err']
+              str = format '≈ %.2f s ± %.2f %s', last['avg'], last['err'], task.worker&.name
+            elsif last['avg']
+              str = format '≈ %.2f s %s', last['avg'], task.worker&.name
+            end
+            ui_task_string(task, 3 + index, x, str: str) if str
           end
         end
       end

data/lib/fasten/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fasten
-  VERSION = '0.5.2'
+  VERSION = '0.5.4'
 end

data/lib/fasten/worker.rb

@@ -8,11 +8,16 @@ module Fasten
     end
   end
 
-  class Worker < Task
+  class Worker
     include Fasten::Logger
+    include Fasten::State
+
+    attr_accessor :executor, :name, :spinner, :child_read, :child_write, :parent_read, :parent_write, :pid, :block, :running_task
 
     def initialize(executor:, name: nil)
-      super executor: executor, name: name, spinner: 0
+      self.executor = executor
+      self.name = name
+      self.spinner = 0
     end
 
     def perform(task)
@@ -48,40 +53,32 @@ module Fasten
     end
 
     def send_request(task)
-      Marshal.dump(Task.new(task.to_h.merge(depends: nil, dependants: nil)), parent_write)
-      self.running_task = task
-      task.worker = self
       task.state = :RUNNING
+      task.worker = self
+      self.running_task = task
+      self.state = :RUNNING
+      Marshal.dump(task, parent_write)
     end
 
     def receive_response
       updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
 
-      %i[ini fin response error].each { |key| running_task[key] = updated_task[key] }
+      %i[state ini fin dif response error].each { |key| running_task.send "#{key}=", updated_task.send(key) }
 
       task = running_task
-      self.running_task = nil
-      task.state = task.error ? :FAIL : :DONE
+      self.running_task = self.state = nil
 
       task
     end
 
     def kill
       log_info 'Removing worker'
-      Process.kill(:KILL, pid)
+      Process.kill :KILL, pid
       close_parent_pipes
     rescue StandardError => error
       log_warn "Ignoring error killing worker #{self}, error: #{error}"
     end
 
-    def idle?
-      running_task.nil?
-    end
-
-    def running?
-      !idle?
-    end
-
     protected
 
     def create_pipes
@@ -127,7 +124,9 @@ module Fasten
       log_ini task, 'perform'
 
       perform(task)
+      task.state = :DONE
     rescue StandardError => error
+      task.state = :FAIL
       task.error = WorkerError.new(error)
     ensure
       log_fin task, 'perform'

data/lib/fasten/yaml.rb

@@ -0,0 +1,48 @@
+module Fasten
+  module Yaml
+    def transform_params(params)
+      params.keys.each do |k|
+        val = params[k]
+
+        if val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
+          val = Regexp.new(match[1])
+        end
+
+        params[k.to_sym] = val
+        params.delete(k)
+      end
+    end
+
+    def load_yaml(path)
+      items = YAML.safe_load(File.read(path)).each do |name, params|
+        if params.is_a? String
+          params = { after: params }
+        elsif params.is_a? Hash
+          transform_params(params)
+        else
+          params = {}
+        end
+
+        add Fasten::Task.new({ name: name }.merge(params))
+      end
+
+      log_info "Loaded #{items.count} tasks from #{path}"
+    end
+
+    def save_yaml(path)
+      keys = %i[after shell]
+
+      items = task_list.map do |task|
+        data = task.to_h.select do |key, _val|
+          keys.include? key
+        end
+
+        [task.name, data]
+      end.to_h
+
+      File.write path, items.to_yaml
+
+      log_info "Loaded #{items.count} tasks into #{path}"
+    end
+  end
+end

data/lib/fasten.rb

@@ -11,12 +11,13 @@ require 'csv'
 require 'hirb'
 require 'parallel'
 
+require 'fasten/state'
 require 'fasten/logger'
 require 'fasten/stats'
 require 'fasten/task'
 require 'fasten/ui'
 require 'fasten/dag'
-require 'fasten/load_save'
+require 'fasten/yaml'
 require 'fasten/executor'
 require 'fasten/worker'
 require 'fasten/version'
@@ -25,9 +26,9 @@ module Fasten
   class << self
     include Fasten::Logger
 
-    def load(path, **options)
+    def from_yaml(path, **options)
       executor = Fasten::Executor.new(**options)
-      executor.load(path)
+      executor.load_yaml(path)
 
       executor
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fasten
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.4
 platform: ruby
 authors:
 - Aldrin Martoq
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-26 00:00:00.000000000 Z
+date: 2018-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -161,8 +161,8 @@ files:
 - lib/fasten.rb
 - lib/fasten/dag.rb
 - lib/fasten/executor.rb
-- lib/fasten/load_save.rb
 - lib/fasten/logger.rb
+- lib/fasten/state.rb
 - lib/fasten/stats.rb
 - lib/fasten/task.rb
 - lib/fasten/ui.rb
@@ -170,6 +170,7 @@ files:
 - lib/fasten/ui/curses.rb
 - lib/fasten/version.rb
 - lib/fasten/worker.rb
+- lib/fasten/yaml.rb
 homepage: https://github.com/a0/fasten/
 licenses:
 - MIT

data/lib/fasten/load_save.rb

@@ -1,73 +0,0 @@
-module Fasten
-  module LoadSave
-    attr_reader :stats_path
-
-    def load(path)
-      items = YAML.safe_load(File.read(path)).each do |name, params|
-        if params.is_a? String
-          params = { after: params }
-        else
-          params&.each do |key, val|
-            next unless val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
-
-            params[key] = Regexp.new(match[1])
-          end
-        end
-
-        add Fasten::Task.new({ name: name }.merge(params || {}))
-      end
-
-      log_info "Loaded #{items.count} tasks from #{path}"
-    end
-
-    def save(path)
-      keys = %i[after shell]
-
-      items = task_list.map do |task|
-        data = task.to_h.select do |key, _val|
-          keys.include? key
-        end
-
-        [task.name, data]
-      end.to_h
-
-      File.write path, items.to_yaml
-
-      log_info "Loaded #{items.count} tasks into #{path}"
-    end
-
-    def load_stats
-      return unless @stats_path && File.exist?(@stats_path)
-
-      self.stats_data = []
-      CSV.foreach(@stats_path, headers: true) do |row|
-        stats_data << row.to_h
-      end
-
-      @task_waiting_list = nil
-    rescue StandardError
-      nil
-    ensure
-      self.stats ||= {}
-    end
-
-    def save_stats
-      return unless @stats_path && stats_data
-
-      CSV.open(@stats_path, 'wb') do |csv|
-        csv << stats_data.first.keys
-
-        stats_data.each do |data|
-          csv << data.values
-        end
-      end
-    end
-
-    def setup_stats(name)
-      @stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME'] && name
-      FileUtils.mkdir_p File.dirname(@stats_path)
-    rescue StandardError
-      @stats_path = nil
-    end
-  end
-end