fasten 0.5.4 → 0.6.0

data/lib/fasten/support/dag.rb

@@ -0,0 +1,130 @@
+module Fasten
+  module Support
+    module DAG
+      attr_reader :task_map, :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
+
+      def initialize_dag
+        @task_map ||= {}
+        @task_list ||= []
+        @task_done_list ||= []
+        @task_error_list ||= []
+        @task_pending_list ||= []
+        @task_running_list ||= []
+
+        nil
+      end
+
+      def add(task)
+        raise "Task '#{task.name}' already defined" if @task_map[task.name]
+
+        @task_map[task.name] = task
+        @task_list << task
+        @task_waiting_list = nil
+      end
+
+      def update_task(task)
+        task.state == :DONE ? update_done_task(task) : update_error_task(task)
+
+        stats_add_entry(task.state, task)
+      end
+
+      def update_done_task(task)
+        @task_done_list << task
+        @task_pending_list.delete task
+        task.dependants.each { |dependant_task| dependant_task.depends.delete task }
+
+        move_pending_to_waiting
+      end
+
+      def update_error_task(task)
+        @task_error_list << task
+        @task_pending_list.delete task
+      end
+
+      def next_task
+        task_waiting_list.shift
+      end
+
+      def task_waiting_list
+        return @task_waiting_list if @task_waiting_list
+
+        reset_tasks
+        setup_tasks_dependencies
+        setup_tasks_scores
+        move_pending_to_waiting
+      end
+
+      protected
+
+      def move_pending_to_waiting
+        move_list = task_pending_list.select { |task| task.depends.count.zero? }
+
+        @task_waiting_list ||= []
+        @task_pending_list -= move_list
+        @task_waiting_list += move_list
+        @task_waiting_list.sort_by!.with_index do |x, index|
+          x.state = :WAIT
+          [-x.run_score, index]
+        end
+      end
+
+      def reset_tasks
+        @task_pending_list.clear
+        @task_done_list.clear
+        @task_error_list.clear
+
+        @task_list.each do |task|
+          task.dependants = []
+          task.depends = []
+
+          if task.state == :DONE
+            @task_done_list << task
+          elsif task.state == :FAIL
+            @task_error_list << task
+          else
+            task.state = :IDLE
+            @task_pending_list << task
+          end
+        end
+      end
+
+      def setup_tasks_dependencies
+        @task_pending_list.each do |task|
+          next unless task.after
+
+          [task.after].flatten.each do |after|
+            after_task = after.is_a?(Task) ? after : @task_map[after]
+            raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
+
+            task.depends << after_task
+            after_task.dependants << task
+          end
+        end
+      end
+
+      def setup_tasks_scores
+        @task_pending_list.each { |task| task.run_score = task.dependants.count }
+      end
+
+      def no_waiting_tasks?
+        task_waiting_list.empty?
+      end
+
+      def no_running_tasks?
+        task_running_list.empty?
+      end
+
+      def tasks_waiting?
+        !task_waiting_list.empty?
+      end
+
+      def tasks_running?
+        !task_running_list.empty?
+      end
+
+      def tasks_failed?
+        !task_error_list.empty?
+      end
+    end
+  end
+end

data/lib/fasten/support/fork_worker.rb

@@ -0,0 +1,107 @@
+module Fasten
+  module Support
+    module ForkWorker
+      attr_accessor :pid
+
+      def start
+        create_pipes
+
+        self.pid = Process.fork do
+          close_parent_pipes
+
+          process_incoming_requests
+        end
+
+        close_child_pipes
+      end
+
+      def kill
+        log_info 'Removing worker'
+        Process.kill :KILL, pid
+      rescue StandardError => error
+        log_warn "Ignoring error killing worker #{self}, error: #{error}"
+      ensure
+        close_parent_pipes
+        close_child_pipes
+      end
+
+      def send_request_to_child(task)
+        task.state = :RUNNING
+        task.worker = self
+        self.running_task = task
+        self.state = :RUNNING
+
+        Marshal.dump(task, parent_write)
+      end
+
+      def receive_response_from_child
+        updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
+
+        %i[state ini fin dif response error].each { |key| running_task.send "#{key}=", updated_task.send(key) }
+
+        task = running_task
+        self.running_task = self.state = nil
+
+        task
+      end
+
+      def receive_request_from_parent
+        Marshal.load(child_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
+      end
+
+      def send_response_to_parent(task)
+        log_info "Sending task response back to runner #{task}"
+
+        data = Marshal.dump(task)
+        child_write.write(data)
+      end
+
+      def create_pipes
+        self.child_read, self.parent_write = IO.pipe
+        self.parent_read, self.child_write = IO.pipe
+      end
+
+      def close_parent_pipes
+        parent_read.close unless parent_read.closed?
+        parent_write.close unless parent_write.closed?
+      end
+
+      def close_child_pipes
+        child_read.close unless child_read.closed?
+        child_write.close unless child_write.closed?
+      end
+
+      def redirect_std(path)
+        logger.reopen($stdout)
+
+        @saved_stdout_instance = $stdout.clone
+        @saved_stderr_instance = $stderr.clone
+        @saved_stdout_constant = STDOUT.clone
+        @saved_stderr_constant = STDERR.clone
+
+        FileUtils.mkdir_p File.dirname(path)
+        @redirect_log = File.new path, 'a'
+        @redirect_log.sync = true
+
+        $stdout.reopen @redirect_log
+        $stderr.reopen @redirect_log
+        STDOUT.reopen @redirect_log
+        STDERR.reopen @redirect_log
+      end
+
+      def restore_std
+        oldverbose = $VERBOSE
+        $VERBOSE = nil
+
+        $stdout = @saved_stdout_instance
+        $stderr = @saved_stderr_instance
+        Object.const_set :STDOUT, @saved_stdout_constant
+        Object.const_set :STDERR, @saved_stderr_constant
+
+        @redirect_log.close
+      ensure
+        $VERBOSE = oldverbose
+      end
+    end
+  end
+end

data/lib/fasten/support/logger.rb

@@ -0,0 +1,57 @@
+require 'binding_of_caller'
+require 'fileutils'
+require 'logger'
+
+module Fasten
+  class << self
+    attr_accessor :logger
+  end
+
+  module Support
+    module Logger
+      attr_accessor :log_file, :logger
+
+      %w[debug info error].each do |method|
+        define_method "log_#{method}" do |msg|
+          dest_logger = logger || Fasten.logger
+          return unless dest_logger.respond_to?(method)
+
+          caller_name = name if respond_to? :name
+          caller_name ||= Kernel.binding.of_caller(1).eval('self').class
+          dest_logger.send(method, caller_name) { msg }
+        end
+      end
+
+      def initialize_logger(log_file: nil)
+        if log_file
+          self.log_file = log_file
+        else
+          log_path ||= "#{fasten_dir}/log/#{kind}/#{name}.log"
+          FileUtils.mkdir_p File.dirname(log_path)
+          self.log_file = File.new(log_path, 'a')
+          self.log_file.sync = true
+        end
+        self.logger = ::Logger.new self.log_file, level: Fasten.logger.level, progname: Fasten.logger.progname
+      end
+
+      def log_ini(object, message = nil)
+        object.ini ||= Time.new
+        log_info "Ini #{object.state} #{object.class} #{object} #{message}"
+      end
+
+      def log_fin(object, message = nil)
+        object.fin ||= Time.new
+        object.dif = object.fin - object.ini
+
+        log_info "Fin #{object.state} #{object.class} #{object} #{message} in #{object.dif}"
+      end
+    end
+  end
+end
+
+Fasten.logger ||=
+  begin
+    Logger.new STDOUT, level: Logger::DEBUG, progname: $PROGRAM_NAME
+  rescue StandardError
+    nil
+  end

data/lib/fasten/support/state.rb

@@ -0,0 +1,32 @@
+module Fasten
+  module Support
+    module State
+      attr_accessor :error, :ini, :fin, :dif, :last
+      attr_writer :state
+
+      def state
+        @state || :IDLE
+      end
+
+      def running?
+        state == :RUNNING
+      end
+
+      def idle?
+        state == :IDLE
+      end
+
+      def pausing?
+        state == :PAUSING
+      end
+
+      def paused?
+        state == :PAUSED
+      end
+
+      def quitting?
+        state == :QUITTING
+      end
+    end
+  end
+end

data/lib/fasten/support/stats.rb

@@ -0,0 +1,137 @@
+require 'csv'
+require 'hirb'
+require 'fileutils'
+
+module Fasten
+  module Support
+    module Stats
+      attr_writer :stats_data, :stats_entries
+      attr_reader :stats_path
+
+      def initialize_stats
+        return unless stats
+
+        @stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME']
+        FileUtils.mkdir_p File.dirname(@stats_path)
+      rescue StandardError
+        @stats_path = nil
+      end
+
+      def load_stats
+        return unless @stats_path && File.exist?(@stats_path)
+
+        self.stats_data = []
+        CSV.foreach(@stats_path, headers: true) do |row|
+          stats_data << row.to_h
+        end
+
+        @task_waiting_list = nil
+      rescue StandardError
+        nil
+      end
+
+      def save_stats
+        return unless @stats_path && stats_data
+
+        keys = %w[state kind name run cnt avg std err]
+
+        CSV.open(@stats_path, 'wb') do |csv|
+          csv << keys
+
+          stats_data.each do |data|
+            csv << keys.map { |i| data[i] }
+          end
+        end
+      end
+
+      def stats_create_entry(state, target)
+        { 'state'  => state.to_s,
+          'kind'   => target.kind,
+          'name'   => target.name,
+          'ini'    => target.ini.to_f,
+          'fin'    => target.fin.to_f,
+          'run'    => target.fin - target.ini,
+          'worker' => target.respond_to?(:worker) ? target.worker.name : nil }
+      end
+
+      def stats_data
+        @stats_data ||= []
+      end
+
+      def stats_entries
+        @stats_entries ||= []
+      end
+
+      def stats_add_entry(state, target)
+        return unless target.ini && target.fin
+
+        entry = stats_create_entry(state, target)
+        stats_data << entry
+        stats_entries << entry
+
+        history = stats_history(entry)
+
+        update_stats(history, entry)
+      end
+
+      FLOAT_FORMATTER = ->(f) { format('%7.3f', f) }
+
+      def stats_table_run
+        sub = stats_entries.select { |x| x['kind'] == 'task' }.map { |x| x['run'] }.sum
+        tot = stats_entries.select { |x| x['kind'] == 'runner' }.map { |x| x['run'] }.sum
+
+        [sub, tot]
+      end
+
+      def split_time(time)
+        sign = time.negative? ? '-' : ''
+        time = -time if time.negative?
+
+        hours, seconds = time.divmod(3600)
+        minutes, seconds = seconds.divmod(60)
+        seconds, decimal = seconds.divmod(1)
+        milliseconds, _ignored = (decimal.round(4) * 1000).divmod(1)
+
+        [sign, hours, minutes, seconds, milliseconds]
+      end
+
+      def hformat(time, total = nil)
+        sign, hours, mins, secs, msecs = split_time time
+
+        str = hours.zero? ? format('%.1s%02d:%02d.%03d', sign, mins, secs, msecs) : format('%.1s%02d:%02d:%02d.%03d', sign, hours, mins, secs, msecs)
+        str += format(' (%.1f%%)', 100.0 * time / total) if total
+
+        str
+      end
+
+      def stats_table
+        sub, tot = stats_table_run
+
+        Hirb::Console.render_output(stats_entries,
+                                    fields: %w[state kind name run cnt avg std err worker], unicode: true, class: 'Hirb::Helpers::AutoTable',
+                                    filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER, 'err' => FLOAT_FORMATTER },
+                                    description: false)
+
+        puts format('∑tasks: %<task>s runner: %<runner>s saved: %<saved>s workers: %<workers>s',
+                    task: hformat(sub), runner: hformat(tot, sub), saved: hformat(sub - tot, sub), workers: workers.to_s)
+      end
+
+      def stats_history(entry)
+        stats_data.select { |e| e['state'] == entry['state'] && e['kind'] == entry['kind'] && e['name'] == entry['name'] }.map { |x| x['run'].to_f }
+      end
+
+      def stats_last(item)
+        return item.last if item.last
+
+        item.last = stats_data.select { |e| e['kind'] == item.kind && e['name'] == item.name }.last || {}
+      end
+
+      def update_stats(history, entry)
+        entry['cnt'] = count = history.size
+        entry['avg'] = avg = history.sum.to_f / count
+        entry['std'] = std = Math.sqrt(history.inject(0.0) { |v, x| v + (x - avg)**2 })
+        entry['err'] = std / Math.sqrt(count) if count.positive?
+      end
+    end
+  end
+end