fasten 0.5.4 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e475b26b22ab143005ca207f29ee7036571b71934a7fa2bba0ea2be74ba1aaab
-  data.tar.gz: 39429fea342cf0c43e1b84cef14e897a1d12f56a5f11e4c555c28176bfa1cdda
+  metadata.gz: 8a62e1fda9d93b3a6740a65e81a53bcb4449910e770572df0b6c1a0d908e70cf
+  data.tar.gz: 8bacfb093fe45101875b161fd9fb8385df565f60a1c5714b1866bb57a5e7eea6
 SHA512:
-  metadata.gz: 7bd2923a0ae1d4cf5f31b36a3999be1bfec058a99acdd381712e91454e2ed9baf7193bb87de835359b300a9317cdd47f2c92e850c54a36e7dcf26dd88776d720
-  data.tar.gz: b4307bc7ee694ad3740b0c0b361d4c5a4888f5e405d32b6aa91ceee8c0023ee1a6421771a6915a2f3916d0610cdf5136f329f47076e33bb066e7e3eb4fb572e3
+  metadata.gz: 5b466b834558bc93de7d1c6cd01829e7930bc266bf8e0a3f34c34d379f5cc8425c0e60595650a1cc03869c1926eaa2993151406b05e51e4bdfd388a432d00394
+  data.tar.gz: 1b47208c3c81702f184ac1878af2ac8fdc9d0390ad926feb7a8f7cb8552cb27a74b4171e372f12da5ec3b36305a229c650d139aa88b07314aaeee16a7bf4925b

data/.rubocop.yml

@@ -14,3 +14,7 @@ Metrics/AbcSize:
   Max: 17
 Metrics/MethodLength:
   Max: 15
+Metrics/ParameterLists:
+  Max: 6
+Layout/AlignHash:
+  EnforcedHashRocketStyle: table

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    fasten (0.5.4)
+    fasten (0.6.0)
       binding_of_caller
-      curses
       hirb
+      os
       parallel
 
 GEM
@@ -19,12 +19,13 @@ GEM
     diff-lcs (1.3)
     hirb (0.7.3)
     jaro_winkler (1.5.1)
-    method_source (0.9.0)
+    method_source (0.9.1)
+    os (1.0.0)
     parallel (1.12.1)
-    parser (2.5.1.2)
+    parser (2.5.3.0)
       ast (~> 2.4.0)
     powerpack (0.1.2)
-    pry (0.11.3)
+    pry (0.12.0)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     rainbow (3.0.0)
@@ -42,14 +43,14 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubocop (0.59.2)
+    rubocop (0.60.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.5, != 2.5.1.1)
       powerpack (~> 0.1)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
+      unicode-display_width (~> 1.4.0)
     ruby-progressbar (1.10.0)
     unicode-display_width (1.4.0)
 
@@ -57,7 +58,8 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 1.17.1)
+  curses
   fasten!
   pry
   rake (~> 10.0)
@@ -65,4 +67,4 @@ DEPENDENCIES
   rubocop
 
 BUNDLED WITH
-   1.16.6
+   1.17.1

data/exe/fasten

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'fasten'
+
+Fasten.invoke

data/fasten.gemspec

@@ -1,6 +1,7 @@
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fasten/version'
+require 'os'
 
 Gem::Specification.new do |spec|
   spec.name          = 'fasten'
@@ -22,15 +23,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'bundler', '~> 1.17.1'
+  spec.add_development_dependency 'curses' unless OS.windows?
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop'
 
   spec.add_runtime_dependency 'binding_of_caller'
-  spec.add_runtime_dependency 'curses'
   spec.add_runtime_dependency 'hirb'
+  spec.add_runtime_dependency 'os'
   spec.add_runtime_dependency 'parallel'
 
   raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.' unless spec.respond_to?(:metadata)

data/lib/fasten.rb

@@ -1,50 +1,105 @@
 # frozen_string_literal: true
 
-require 'English'
-require 'yaml'
-require 'binding_of_caller'
-require 'logger'
-require 'ostruct'
-require 'curses'
-require 'fileutils'
-require 'csv'
-require 'hirb'
-require 'parallel'
-
-require 'fasten/state'
-require 'fasten/logger'
-require 'fasten/stats'
+require 'optparse'
 require 'fasten/task'
-require 'fasten/ui'
-require 'fasten/dag'
-require 'fasten/yaml'
-require 'fasten/executor'
+require 'fasten/runner'
 require 'fasten/worker'
 require 'fasten/version'
 
 module Fasten
   class << self
-    include Fasten::Logger
+    def runner_from_yaml(path, **options)
+      runner = Fasten::Runner.new(**options)
+      runner.load_yaml(path)
 
-    def from_yaml(path, **options)
-      executor = Fasten::Executor.new(**options)
-      executor.load_yaml(path)
-
-      executor
+      runner
     end
 
     def map(list, **options, &block)
-      executor = Fasten::Executor.new(**options)
-      executor.block = block
+      runner(**options).map(list, &block)
+    end
+
+    def runner(**options)
+      @runner ||= Fasten::Runner.new(**options)
+    end
+
+    def cleanup
+      @runner = nil
+    end
 
-      list.each do |item|
-        executor.add Fasten::Task.new name: item.to_s, request: item
+    def reconfigure(**options)
+      runner.reconfigure(**options)
+    end
+
+    def register(**options, &block)
+      runner(**options).register(&block)
+    end
+
+    def load_fasten(args)
+      args.each do |path|
+        if File.directory? path
+          items = Dir["#{path}/*_fasten.rb"]
+          items.each do |item|
+            puts "Fasten: loading #{item}"
+            load item
+          end
+        elsif File.file? path
+          puts "Fasten: loading #{path}"
+          load path
+        else
+          STDERR.puts "Fasten: file/folder not found: #{path}"
+          exit 1
+        end
       end
+    end
+
+    def opt_parser # rubocop:disable Metrics/MethodLength
+      return @opt_parser if defined? @opt_parser
+
+      @options = { developer: false }
+
+      @opt_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: #{$PROGRAM_NAME} [options] FILE/DIRECTORY"
+        opts.separator ''
+        opts.separator 'Options'
+
+        opts.on '-n', '--name NAME', String, "Name of this job" do |name|
+          @options[:name] = name
+        end
+        opts.on '-w', '--workers WORKERS', Numeric, "Number of processes/threads to use (#{Parallel.physical_processor_count} on this machine)" do |workers|
+          @options[:workers] = workers
+        end
+        opts.on '-t', '--threads' do
+          @options[:use_threads] = true
+        end
+        opts.on '-p', '--processes' do
+          @options[:use_threads] = false
+        end
+        opts.on '-v', '--version' do
+          puts Fasten::VERSION
+          exit 0
+        end
+        opts.on_tail '-h', '--help' do
+          puts opts
+          exit 0
+        end
+      end
+    end
+
+    def show_help
+      puts opt_parser
+      exit 1
+    end
+
+    def invoke
+      opt_parser.parse!
+
+      show_help if ARGV.length.zero?
 
-      executor.perform
-      executor.stats_table
+      runner(@options)
+      load_fasten ARGV
 
-      executor.task_list.map(&:response)
+      runner.perform
     end
   end
 end

data/lib/fasten/{executor.rb → runner.rb}

@@ -1,33 +1,58 @@
+require 'English'
+require 'parallel'
+require 'pry'
+require 'os'
+
+require 'fasten/support/dag'
+require 'fasten/support/logger'
+require 'fasten/support/state'
+require 'fasten/support/stats'
+require 'fasten/support/ui'
+require 'fasten/support/yaml'
+require 'fasten/timeout_queue'
+
 module Fasten
-  class Executor
-    include Fasten::Logger
-    include Fasten::State
-    include Fasten::DAG
-    include Fasten::UI
-    include Fasten::Yaml
-    include Fasten::Stats
-
-    attr_accessor :name, :workers, :worker_class, :pid, :fasten_dir, :developer, :stats, :worker_list, :block
-
-    def initialize(name: nil, developer: STDIN.tty? && STDOUT.tty?, workers: Parallel.physical_processor_count, worker_class: Worker, fasten_dir: '.fasten')
-      self.stats = name && true
-      self.name = name || "#{self.class} #{$PID}"
-      self.state = :IDLE
-      self.workers = workers
-      self.worker_class = worker_class
-      self.fasten_dir = fasten_dir
-      self.developer = developer
+  class Runner
+    include Fasten::Support::DAG
+    include Fasten::Support::Logger
+    include Fasten::Support::State
+    include Fasten::Support::Stats
+    include Fasten::Support::UI
+    include Fasten::Support::Yaml
+
+    attr_accessor :name, :workers, :worker_class, :fasten_dir, :developer, :stats, :worker_list, :use_threads, :queue
+
+    def initialize(name: nil, developer: STDIN.tty? && STDOUT.tty?, workers: Parallel.physical_processor_count, worker_class: Worker, fasten_dir: '.fasten', use_threads: !OS.posix?)
+      reconfigure(name: name, developer: developer, workers: workers, worker_class: worker_class, fasten_dir: fasten_dir, use_threads: use_threads)
+    end
+
+    def reconfigure(**options)
+      self.stats = options[:name] && true if options[:name] || options.key?(:stats)
+      self.name = options[:name] || "#{self.class} #{$PID}" if options[:name]
+      self.workers = options[:workers] if options[:workers]
+      self.worker_class = options[:worker_class] if options[:worker_class]
+      self.fasten_dir = options[:fasten_dir] if options[:fasten_dir]
+      self.developer = options[:developer] if options[:developer]
+      self.use_threads = options[:use_threads] if options[:use_threads]
 
       initialize_dag
       initialize_stats
       initialize_logger
 
-      self.worker_list = []
+      self.worker_list ||= []
+    end
+
+    def task(name, **opts, &block)
+      add Task.new(name: name, **opts, block: block)
+    end
+
+    def register(&block)
+      instance_eval(&block)
     end
 
     def perform
-      log_ini self, running_counters
       self.state = :RUNNING
+      log_ini self, running_counters
       load_stats
 
       run_ui do
@@ -38,9 +63,20 @@ module Fasten
       log_fin self, running_counters
 
       stats_add_entry(state, self)
+    ensure
       save_stats
     end
 
+    def map(list, &block)
+      list.each do |item|
+        add Fasten::Task.new name: item.to_s, request: item, block: block
+      end
+
+      perform
+
+      task_list.map(&:response)
+    end
+
     def done_counters
       "#{task_done_list.count}/#{task_list.count}"
     end
@@ -82,22 +118,53 @@ module Fasten
     end
 
     def wait_for_running_tasks
+      use_threads ? wait_for_running_tasks_thread : wait_for_running_tasks_fork
+    end
+
+    def wait_for_running_tasks_thread
+      self.queue ||= TimeoutQueue.new
+
+      while should_wait_for_running_tasks?
+        ui.update
+
+        tasks = queue.receive_with_timeout(0.5)
+
+        receive_workers_tasks_thread(tasks)
+      end
+
+      ui.update
+    end
+
+    def receive_workers_tasks_thread(tasks)
+      tasks&.each do |task|
+        task_running_list.delete task
+
+        task.worker.running_task = task.worker.state = nil
+
+        update_task task
+
+        log_fin task, done_counters
+        ui.force_clear
+      end
+    end
+
+    def wait_for_running_tasks_fork
       while should_wait_for_running_tasks?
         ui.update
         reads = worker_list.map(&:parent_read)
-        reads, _writes, _errors = IO.select(reads, [], [], 1)
+        reads, _writes, _errors = IO.select(reads, [], [], 0.5)
 
-        receive_workers_tasks(reads)
+        receive_workers_tasks_fork(reads)
       end
 
       ui.update
     end
 
-    def receive_workers_tasks(reads)
+    def receive_workers_tasks_fork(reads)
       reads&.each do |read|
         next unless (worker = worker_list.find { |item| item.parent_read == read })
 
-        task = worker.receive_response
+        task = worker.receive_response_from_child
 
         task_running_list.delete task
 
@@ -151,9 +218,8 @@ module Fasten
 
       unless worker
         @worker_id = (@worker_id || 0) + 1
-        worker = worker_class.new executor: self, name: "#{worker_class}-#{format '%02X', @worker_id}"
-        worker.block = block if block
-        worker.fork
+        worker = worker_class.new runner: self, name: "#{worker_class}-#{format '%02X', @worker_id}", use_threads: use_threads
+        worker.start
         worker_list << worker
 
         log_info "Worker created: #{worker}"
@@ -170,7 +236,7 @@ module Fasten
 
         task = next_task
         log_ini task, "on worker #{worker}"
-        worker.send_request(task)
+        worker.send_request_to_child(task)
         task_running_list << task
 
         ui.force_clear
@@ -183,5 +249,13 @@ module Fasten
 
       ui.force_clear
     end
+
+    def kind
+      'runner'
+    end
+
+    def to_s
+      name
+    end
   end
 end

data/lib/fasten/std_thread_proxy.rb

@@ -0,0 +1,56 @@
+module Fasten
+  class StdThreadProxy
+    def initialize(original)
+      @original = original
+    end
+
+    def respond_to?(name, include_private = false)
+      target = Thread.current[:FASTEN_STD_THREAD_PROXY] || @original
+      target.send :respond_to?, name, include_private
+    end
+
+    private
+
+    def respond_to_missing?(name, include_private = false)
+      target = Thread.current[:FASTEN_STD_THREAD_PROXY] || @original
+      target.send :respond_to_missing?, name, include_private
+    end
+
+    def method_missing(method, *args, &block) # rubocop:disable MethodMissingSuper
+      target = Thread.current[:FASTEN_STD_THREAD_PROXY] || @original
+      target.send method, *args, &block
+    end
+
+    class << self
+      def install
+        return if @installed
+
+        oldverbose = $VERBOSE
+        $VERBOSE = nil
+
+        Object.const_set :STDOUT, StdThreadProxy.new(STDOUT)
+        Object.const_set :STDERR, StdThreadProxy.new(STDERR)
+        $stdout = StdThreadProxy.new $stdout
+        $stderr = StdThreadProxy.new $stderr
+
+        @installed = true
+      ensure
+        $VERBOSE = oldverbose
+      end
+
+      def thread_io=(io)
+        Thread.current[:FASTEN_STD_THREAD_PROXY] = io
+      end
+
+      def thread_io
+        Thread.current[:FASTEN_STD_THREAD_PROXY]
+      end
+
+      def uninstall
+        return unless @installed
+
+        @installed = nil
+      end
+    end
+  end
+end