fasten 0.5.4 → 0.6.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fasten
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
 platform: ruby
 authors:
 - Aldrin Martoq
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-30 00:00:00.000000000 Z
+date: 2018-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 1.17.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 1.17.1
+- !ruby/object:Gem::Dependency
+  name: curses
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: curses
+  name: hirb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: hirb
+  name: os
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -139,7 +153,8 @@ dependencies:
 description: Fasten your seatbelts! Run jobs in parallel, intelligently.
 email:
 - contacto@a0.cl
-executables: []
+executables:
+- fasten
 extensions: []
 extra_rdoc_files: []
 files:
@@ -157,20 +172,25 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- exe/fasten
 - fasten.gemspec
 - lib/fasten.rb
-- lib/fasten/dag.rb
-- lib/fasten/executor.rb
-- lib/fasten/logger.rb
-- lib/fasten/state.rb
-- lib/fasten/stats.rb
+- lib/fasten/runner.rb
+- lib/fasten/std_thread_proxy.rb
+- lib/fasten/support/dag.rb
+- lib/fasten/support/fork_worker.rb
+- lib/fasten/support/logger.rb
+- lib/fasten/support/state.rb
+- lib/fasten/support/stats.rb
+- lib/fasten/support/thread_worker.rb
+- lib/fasten/support/ui.rb
+- lib/fasten/support/yaml.rb
 - lib/fasten/task.rb
-- lib/fasten/ui.rb
+- lib/fasten/timeout_queue.rb
 - lib/fasten/ui/console.rb
 - lib/fasten/ui/curses.rb
 - lib/fasten/version.rb
 - lib/fasten/worker.rb
-- lib/fasten/yaml.rb
 homepage: https://github.com/a0/fasten/
 licenses:
 - MIT

data/lib/fasten/dag.rb

@@ -1,137 +0,0 @@
-module Fasten
-  module DAG
-    attr_reader :task_map, :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
-
-    def initialize_dag
-      @task_map = {}
-      @task_list = []
-      @task_done_list = []
-      @task_error_list = []
-      @task_pending_list = []
-      @task_running_list = []
-    end
-
-    def add(task)
-      raise "Task '#{task.name}' already defined" if @task_map[task.name]
-
-      @task_map[task.name] = task
-      @task_list << task
-      @task_waiting_list = nil
-    end
-
-    def update_task(task)
-      if task.state == :DONE
-        update_done_task task
-      else
-        update_error_task task
-      end
-
-      stats_add_entry(task.state, task)
-    end
-
-    def update_done_task(task)
-      @task_done_list << task
-      @task_pending_list.delete task
-      task.dependants.each do |dependant_task|
-        dependant_task.depends.delete task
-      end
-
-      move_pending_to_waiting
-    end
-
-    def update_error_task(task)
-      @task_error_list << task
-      @task_pending_list.delete task
-    end
-
-    def next_task
-      task_waiting_list.shift
-    end
-
-    def task_waiting_list
-      return @task_waiting_list if @task_waiting_list
-
-      reset_tasks
-      setup_tasks_dependencies
-      setup_tasks_scores
-      move_pending_to_waiting
-    end
-
-    protected
-
-    def move_pending_to_waiting
-      move_list = task_pending_list.select do |task|
-        task.depends.count.zero?
-      end
-
-      @task_waiting_list ||= []
-      @task_pending_list -= move_list
-      @task_waiting_list += move_list
-      @task_waiting_list.sort_by!.with_index do |x, index|
-        x.state = :WAIT
-        [-x.run_score, index]
-      end
-    end
-
-    def reset_tasks
-      @task_pending_list.clear
-      @task_done_list.clear
-      @task_error_list.clear
-
-      @task_list.each do |task|
-        task.dependants = []
-        task.depends = []
-        task.run_score = 0
-
-        if task.state == :DONE
-          @task_done_list << task
-        elsif task.state == :FAIL
-          @task_error_list << task
-        else
-          task.state = :IDLE
-          @task_pending_list << task
-        end
-      end
-    end
-
-    def setup_tasks_dependencies
-      @task_pending_list.each do |task|
-        next unless task.after
-
-        [task.after].flatten.each do |after|
-          after_task = after.is_a?(Task) ? after : @task_map[after]
-          raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
-
-          task.depends << after_task
-          after_task.dependants << task
-        end
-      end
-    end
-
-    def setup_tasks_scores
-      @task_pending_list.each do |task|
-        task.run_score += task.dependants.count
-      end
-    end
-
-    def no_waiting_tasks?
-      task_waiting_list.empty?
-    end
-
-    def no_running_tasks?
-      task_running_list.empty?
-    end
-
-    def tasks_waiting?
-      !task_waiting_list.empty?
-    end
-
-    def tasks_running?
-      !task_running_list.empty?
-    end
-
-    def tasks_failed?
-      !task_error_list.empty?
-    end
-  end
-end

data/lib/fasten/logger.rb

@@ -1,62 +0,0 @@
-module Fasten
-  class << self
-    attr_accessor :logger
-  end
-
-  module Logger
-    attr_accessor :log_file
-
-    %w[debug info error].each do |method|
-      define_method "log_#{method}" do |msg|
-        return unless Fasten.logger.respond_to?(method)
-
-        caller_name = name if respond_to? :name
-        caller_name ||= Kernel.binding.of_caller(1).eval('self').class
-        Fasten.logger.send(method, caller_name) { msg }
-      end
-    end
-
-    def initialize_logger
-      log_path = "#{fasten_dir}/log/executor/#{name}.log"
-      FileUtils.mkdir_p File.dirname(log_path)
-      self.log_file = File.new(log_path, 'a')
-      Fasten.logger.reopen log_file
-    end
-
-    def log_ini(object, message = nil)
-      object.ini ||= Time.new
-      log_info "Ini #{object.state} #{object.class} #{object} #{message}"
-    end
-
-    def log_fin(object, message = nil)
-      object.fin ||= Time.new
-      object.dif = object.fin - object.ini
-
-      log_info "Fin #{object.state} #{object.class} #{object} #{message} in #{object.dif}"
-    end
-
-    def redirect_std(path)
-      @saved_stdout = $stdout.clone
-      @saved_stderr = $stderr.clone
-
-      FileUtils.mkdir_p File.dirname(path)
-      log = File.new path, 'a'
-      log.sync = true
-
-      $stdout.reopen log
-      $stderr.reopen log
-    end
-
-    def restore_std
-      $stdout.reopen(@saved_stdout)
-      $stderr.reopen(@saved_stderr)
-    end
-  end
-end
-
-Fasten.logger ||=
-  begin
-    Logger.new(STDOUT, level: Logger::DEBUG, progname: $PROGRAM_NAME)
-  rescue StandardError
-    nil
-  end

data/lib/fasten/state.rb

@@ -1,30 +0,0 @@
-module Fasten
-  module State
-    attr_accessor :error, :ini, :fin, :dif, :last
-    attr_writer :state
-
-    def state
-      @state || :IDLE
-    end
-
-    def running?
-      state == :RUNNING
-    end
-
-    def idle?
-      state == :IDLE
-    end
-
-    def pausing?
-      state == :PAUSING
-    end
-
-    def paused?
-      state == :PAUSED
-    end
-
-    def quitting?
-      state == :QUITTING
-    end
-  end
-end

data/lib/fasten/stats.rb

@@ -1,137 +0,0 @@
-module Fasten
-  module Stats
-    attr_writer :stats_data, :stats_entries
-    attr_reader :stats_path
-
-    def initialize_stats
-      return unless stats
-
-      @stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME']
-      FileUtils.mkdir_p File.dirname(@stats_path)
-    rescue StandardError
-      @stats_path = nil
-    end
-
-    def load_stats
-      return unless @stats_path && File.exist?(@stats_path)
-
-      self.stats_data = []
-      CSV.foreach(@stats_path, headers: true) do |row|
-        stats_data << row.to_h
-      end
-
-      @task_waiting_list = nil
-    rescue StandardError
-      nil
-    ensure
-      self.stats ||= {}
-    end
-
-    def save_stats
-      return unless @stats_path && stats_data
-
-      keys = %w[state kind name run cnt avg std err]
-
-      CSV.open(@stats_path, 'wb') do |csv|
-        csv << keys
-
-        stats_data.each do |data|
-          csv << keys.map { |i| data[i] }
-        end
-      end
-    end
-
-    def stats_create_entry(state, target)
-      { 'state' => state.to_s,
-        'kind'  => stats_kind_for(target),
-        'name'  => target.name,
-        'ini'   => target.ini.to_f,
-        'fin'   => target.fin.to_f,
-        'run'   => target.fin - target.ini,
-        'worker' => target.respond_to?(:worker) ? target.worker.name : nil }
-    end
-
-    def stats_data
-      @stats_data ||= []
-    end
-
-    def stats_entries
-      @stats_entries ||= []
-    end
-
-    def stats_kind_for(object)
-      object.is_a?(Fasten::Executor) ? 'executor' : 'task'
-    end
-
-    def stats_add_entry(state, target)
-      return unless target.ini && target.fin
-
-      entry = stats_create_entry(state, target)
-      stats_data << entry
-      stats_entries << entry
-
-      history = stats_history(entry)
-
-      update_stats(history, entry)
-    end
-
-    FLOAT_FORMATTER = ->(f) { format('%7.3f', f) }
-
-    def stats_table_run
-      sub = stats_entries.select { |x| x['kind'] == 'task' }.map { |x| x['run'] }.sum
-      tot = stats_entries.select { |x| x['kind'] == 'executor' }.map { |x| x['run'] }.sum
-
-      [sub, tot]
-    end
-
-    def split_time(time)
-      sign = time.negative? ? '-' : ''
-      time = -time if time.negative?
-
-      hours, seconds = time.divmod(3600)
-      minutes, seconds = seconds.divmod(60)
-      seconds, decimal = seconds.divmod(1)
-      milliseconds, _ignored = (decimal.round(4) * 1000).divmod(1)
-
-      [sign, hours, minutes, seconds, milliseconds]
-    end
-
-    def hformat(time, total = nil)
-      sign, hours, minutes, seconds, milliseconds = split_time time
-
-      str = hours.zero? ? format('%.1s%02d:%02d.%03d', sign, minutes, seconds, milliseconds) : format('%.1s%02d:%02d:%02d.%03d', sign, hours, minutes, seconds, milliseconds)
-      str += format(' (%.1f%%)', 100.0 * time / total) if total
-
-      str
-    end
-
-    def stats_table
-      sub, tot = stats_table_run
-
-      Hirb::Console.render_output(stats_entries,
-                                  fields: %w[state kind name run cnt avg std err worker], unicode: true, class: 'Hirb::Helpers::AutoTable',
-                                  filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER, 'err' => FLOAT_FORMATTER },
-                                  description: false)
-
-      puts format('∑tasks: %<task>s ∑executed: %<executed>s saved: %<saved>s workers: %<workers>s',
-                  task: hformat(sub), executed: hformat(tot, sub), saved: hformat(sub - tot, sub), workers: workers.to_s)
-    end
-
-    def stats_history(entry)
-      stats_data.select { |e| e['state'] == entry['state'] && e['kind'] == entry['kind'] && e['name'] == entry['name'] }
-    end
-
-    def stats_last(item)
-      return item.last if item.last
-
-      item.last = stats_data.select { |e| e['kind'] == stats_kind_for(item) && e['name'] == item.name }.last || {}
-    end
-
-    def update_stats(history, entry)
-      entry['cnt'] = count = history.size
-      entry['avg'] = avg = history.inject(0.0) { |s, x| s + x['run'].to_f } / count
-      entry['std'] = std = Math.sqrt(history.inject(0.0) { |v, x| v + (x['run'].to_f - avg)**2 })
-      entry['err'] = std / Math.sqrt(count) if count.positive?
-    end
-  end
-end