fasten 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ca1b131f4ee26ffb7f5dcaa5408e314988c3568d155b9cb804daba84c588f90
-  data.tar.gz: 6bd08febfd39b5e0ab11c95e6e687acdb6778c67f493469033e49364f47b6012
+  metadata.gz: 3b5fa7853f4b7c0e9a187bb5f76f59019e7db4defdbc68d059ca71a90ad38b8b
+  data.tar.gz: 91db6845b40c890928b3f217e5d91ab88f127187b95257a4863533db92dc0840
 SHA512:
-  metadata.gz: 5fa5a15d9c430cbbe53fee25917c9dcaba4dfb40e70b6b5323d1a46036d0e2a53fdc1d4960226bddf14a76946499fcc8e777b9431596c7c92bfd261093476ed3
-  data.tar.gz: 4d0596481e975f9c4406f1c46ce707cde8f1f9d11b4089e94a825fe5281b3139e6a9fdf82ac3e905b606ef16a6157019aa04ee3bdd7a3687ebe5310d2303ede3
+  metadata.gz: 184e4868d0cc1b5e24ae365e636e7fb988c656665c05420abce2551024143b53857cf32ebb1e717cadedfba655cae42079d1044aa5ae8f82872478d762be8190
+  data.tar.gz: c7f0c1c4d1a8a23ee97be8916de501eacda6a70b526b387fbabbac130743b6aadcfccba1d626581cf0ebe860006cecb58cb7729cd48a4d8aa9d86633ef9a61ab

data/.gitignore

@@ -10,3 +10,4 @@
 # rspec failure tracking
 .rspec_status
 .fasten
+*.testfile

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fasten (0.2.0)
+    fasten (0.4.0)
       binding_of_caller
       curses
 

data/lib/fasten.rb

@@ -6,11 +6,15 @@ require 'binding_of_caller'
 require 'logger'
 require 'ostruct'
 require 'curses'
+require 'fileutils'
+require 'csv'
 
 require 'fasten/log_support'
+require 'fasten/stats'
 require 'fasten/task'
 require 'fasten/ui'
 require 'fasten/dag'
+require 'fasten/load_save'
 require 'fasten/executor'
 require 'fasten/worker'
 require 'fasten/version'
@@ -26,20 +30,4 @@ module Fasten
       executor
     end
   end
-
-  class Executor
-    def load(path)
-      items = YAML.safe_load(File.read(path)).each do |name, params|
-        params.each do |key, val|
-          next unless val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
-
-          params[key] = Regexp.new(match[1])
-        end
-
-        add Fasten::Task.new({ name: name }.merge(params))
-      end
-
-      log_info "Loaded #{items.count} tasks from #{path}"
-    end
-  end
 end

data/lib/fasten/dag.rb

@@ -1,6 +1,6 @@
 module Fasten
   module DAG
-    attr_reader :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
+    attr_reader :task_map, :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
 
     def initialize_dag
       @task_map = {}
@@ -20,11 +20,13 @@ module Fasten
     end
 
     def update_task(task)
-      if task.error
-        update_error_task task
-      else
+      if task.state == :DONE
         update_done_task task
+      else
+        update_error_task task
       end
+
+      stats_add_entry(self, task.state, task)
     end
 
     def update_done_task(task)
@@ -43,7 +45,7 @@ module Fasten
     end
 
     def next_task
-      task_waiting_list.pop
+      task_waiting_list.sort_by!(&:run_score).pop
     end
 
     def task_waiting_list
@@ -51,6 +53,7 @@ module Fasten
 
       reset_tasks
       setup_tasks_dependencies
+      setup_tasks_scores
       move_pending_to_waiting
     end
 
@@ -72,7 +75,7 @@ module Fasten
       @task_list.each do |task|
         task.dependants = []
         task.depends = []
-        task.level = 0
+        task.run_score = 0
         task.done ? @task_done_list << task : @task_pending_list << task
       end
     end
@@ -86,12 +89,17 @@ module Fasten
           raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
 
           task.depends << after_task
-          task.level += 1
           after_task.dependants << task
         end
       end
     end
 
+    def setup_tasks_scores
+      @task_pending_list.each do |task|
+        task.run_score += task.dependants.count
+      end
+    end
+
     def no_waiting_tasks?
       task_waiting_list.empty?
     end

data/lib/fasten/executor.rb

@@ -3,8 +3,11 @@ module Fasten
     include Fasten::LogSupport
     include Fasten::DAG
     include Fasten::UI
+    include Fasten::LoadSave
+    include Fasten::Stats
 
     def initialize(name: nil, workers: 8, worker_class: Fasten::Worker, fasten_dir: '.fasten')
+      setup_stats(name)
       super name: name || "#{self.class} #{$PID}", workers: workers, pid: $PID, state: :IDLE, worker_class: worker_class, fasten_dir: fasten_dir
       initialize_dag
 
@@ -16,22 +19,26 @@ module Fasten
     end
 
     def perform
-      log_ini self, running_stats
+      log_ini self, running_counters
       self.state = :RUNNING
+      load_stats
 
       run_ui do
         perform_loop
       end
 
-      self.state = :IDLE
-      log_fin self, running_stats
+      self.state = task_list.map(&:state).all?(:DONE) ? :DONE : :FAIL
+      log_fin self, running_counters
+
+      stats_add_entry(self, state, self)
+      save_stats
     end
 
-    def done_stats
+    def done_counters
       "#{task_done_list.count}/#{task_list.count}"
     end
 
-    def running_stats
+    def running_counters
       "#{task_done_list.count + task_running_list.count}/#{task_list.count}"
     end
 
@@ -68,7 +75,7 @@ module Fasten
 
         update_task task
 
-        log_fin task, done_stats
+        log_fin task, done_counters
       end
     end
 

data/lib/fasten/load_save.rb

@@ -0,0 +1,69 @@
+module Fasten
+  module LoadSave
+    attr_reader :stats_path
+
+    def load(path)
+      items = YAML.safe_load(File.read(path)).each do |name, params|
+        params.each do |key, val|
+          next unless val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
+
+          params[key] = Regexp.new(match[1])
+        end
+
+        add Fasten::Task.new({ name: name }.merge(params))
+      end
+
+      log_info "Loaded #{items.count} tasks from #{path}"
+    end
+
+    def save(path)
+      keys = %i[after shell]
+
+      items = task_list.map do |task|
+        data = task.to_h.select do |key, _val|
+          keys.include? key
+        end
+
+        [task.name, data]
+      end.to_h
+
+      File.write path, items.to_yaml
+
+      log_info "Loaded #{items.count} tasks into #{path}"
+    end
+
+    def load_stats
+      return unless @stats_path && File.exist?(@stats_path)
+
+      self.stats_data = []
+      CSV.foreach(@stats_path, headers: true) do |row|
+        stats_data << row.to_h
+      end
+
+      @task_waiting_list = nil
+    rescue StandardError
+      nil
+    ensure
+      self.stats ||= {}
+    end
+
+    def save_stats
+      return unless @stats_path && stats_data
+
+      CSV.open(@stats_path, 'wb') do |csv|
+        csv << stats_data.first.keys
+
+        stats_data.each do |data|
+          csv << data.values
+        end
+      end
+    end
+
+    def setup_stats(name)
+      @stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME'] && name
+      FileUtils.mkdir_p File.dirname(@stats_path)
+    rescue StandardError
+      @stats_path = nil
+    end
+  end
+end

data/lib/fasten/log_support.rb

@@ -32,6 +32,7 @@ module Fasten
 
       FileUtils.mkdir_p File.dirname(path)
       log = File.new path, 'a'
+      log.sync = true
 
       $stdout.reopen log
       $stderr.reopen log

data/lib/fasten/stats.rb

@@ -0,0 +1,39 @@
+module Fasten
+  module Stats
+    def stats_create_entry(state, target)
+      {
+        'state' => state.to_s,
+        'kind' => target.is_a?(Fasten::Executor) ? 'executor' : 'task',
+        'name' => target.name,
+        'ini' => target.ini.to_f,
+        'fin' => target.fin.to_f,
+        'run' => target.fin - target.ini
+      }
+    end
+
+    def stats_add_entry(object, state, target)
+      return unless target.ini && target.fin
+
+      entry = stats_create_entry(state, target)
+      object.stats_data ||= []
+      object.stats_data << entry
+
+      history = stats_history(object, entry)
+
+      update_avg(history, entry)
+      update_std(history, entry)
+    end
+
+    def stats_history(object, entry)
+      object.stats_data.select { |e| e['state'] == entry['state'] && e['kind'] == entry['kind'] && e['name'] == entry['name'] }
+    end
+
+    def update_avg(history, entry)
+      entry['avg'] = history.inject(0.0) { |s, x| s + x['run'].to_f } / history.size
+    end
+
+    def update_std(history, entry)
+      entry['std'] = Math.sqrt(history.inject(0.0) { |v, x| v + (x['run'].to_f - entry['avg'])**2 })
+    end
+  end
+end

data/lib/fasten/ui.rb

@@ -108,7 +108,7 @@ module Fasten
       when :RUNNING
         attrs = color_pair(1) | A_TOP
         icon = SPINNER_STR[task.worker&.spinner] if icon
-      when :ERROR
+      when :FAIL
         attrs = color_pair(3)
         icon = '✘︎' if icon
       when :DONE
@@ -143,14 +143,15 @@ module Fasten
       ui_progressbar(2, col_ini, col_fin, count_done, count_total)
 
       max = 2
-      task_list.each_with_index do |task, index|
+      list = task_list.sort_by(&:run_score)
+      list.each_with_index do |task, index|
         next if 3 + index >= ui_rows
 
         x = ui_task_string(task, 3 + index, 2, icon: true)
         max = x if x > max
       end
 
-      task_list.each_with_index do |task, index|
+      list.each_with_index do |task, index|
         next if 3 + index >= ui_rows || task.depends.nil? || task.depends.empty?
 
         setpos 3 + index, max

data/lib/fasten/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fasten
-  VERSION = '0.2.0'
+  VERSION = '0.4.0'
 end

data/lib/fasten/worker.rb

@@ -30,13 +30,13 @@ module Fasten
     end
 
     def receive
-      updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad
+      updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
 
       %i[ini fin response error].each { |key| running_task[key] = updated_task[key] }
 
       task = running_task
       self.running_task = nil
-      task.state = task.error ? :ERROR : :DONE
+      task.state = task.error ? :FAIL : :DONE
 
       task
     end
@@ -77,7 +77,7 @@ module Fasten
     def process_incoming_tasks
       log_ini self, 'process_incoming_tasks'
 
-      while (object = Marshal.load(child_read)) # rubocop:disable Security/MarshalLoad
+      while (object = Marshal.load(child_read)) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
         run_task(object) if object.is_a? Fasten::Task
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fasten
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Aldrin Martoq
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-14 00:00:00.000000000 Z
+date: 2018-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -133,7 +133,9 @@ files:
 - lib/fasten.rb
 - lib/fasten/dag.rb
 - lib/fasten/executor.rb
+- lib/fasten/load_save.rb
 - lib/fasten/log_support.rb
+- lib/fasten/stats.rb
 - lib/fasten/task.rb
 - lib/fasten/ui.rb
 - lib/fasten/version.rb