fasten 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a62e1fda9d93b3a6740a65e81a53bcb4449910e770572df0b6c1a0d908e70cf
-  data.tar.gz: 8bacfb093fe45101875b161fd9fb8385df565f60a1c5714b1866bb57a5e7eea6
+  metadata.gz: 2d815525a239cf976ae822a2a08d5c3be07ec006c07c7af991c68ce9bcf19b56
+  data.tar.gz: a9be85ef4553f17a9629315e55e6beb1dd28ed1db4c447964909826e5cd2650e
 SHA512:
-  metadata.gz: 5b466b834558bc93de7d1c6cd01829e7930bc266bf8e0a3f34c34d379f5cc8425c0e60595650a1cc03869c1926eaa2993151406b05e51e4bdfd388a432d00394
-  data.tar.gz: 1b47208c3c81702f184ac1878af2ac8fdc9d0390ad926feb7a8f7cb8552cb27a74b4171e372f12da5ec3b36305a229c650d139aa88b07314aaeee16a7bf4925b
+  metadata.gz: 3278db70005e8503299247c54eea40b4469de82e8729a4a839aa6d166070a96c4a71d1284d5bd3ab6702b19d080dddf74ef7a29ccc7a37a4308b452c2e854dbe
+  data.tar.gz: 43cc2129b54bfd13f4863cbc528e6d00ccf2a016976b389ff7a9dc6fbceff14f68e5b0d8ce555a28c6692ecb41f1ebfbfcc93dcaf0db4f9e1b21d3ad7de75464

data/.gitignore

@@ -11,3 +11,4 @@
 .rspec_status
 .fasten
 *.testfile
+.DS_Store

data/.rubocop.yml

@@ -15,6 +15,6 @@ Metrics/AbcSize:
 Metrics/MethodLength:
   Max: 15
 Metrics/ParameterLists:
-  Max: 6
+  Max: 8
 Layout/AlignHash:
   EnforcedHashRocketStyle: table

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fasten (0.6.0)
+    fasten (0.7.0)
       binding_of_caller
       hirb
       os

data/README.md

@@ -1,24 +1,57 @@
-# Fasten your seatbelts!
-
-FIXME: Add intro here
-
-## Roadmap
-
-- [x] define task to be run to reach a goal, with dependencies (similar to rake, make, and others)
-- [x] run task in parallel by default
-- [x] dynamicly change the number of worker processes
-- [x] allow to programatically define new tasks
-- [ ] allow to programatically redefine existing tasks
-- [x] keep each task log in a separate file, to easily debug errors
-- [x] early stop in case of failure
-- [x] for non-tty execution, report with a simple progress log in STDOUT
-- [ ] calculate ETA of script, based on previos executions of the same script
-- [ ] calculate ETA without previous information
-- [ ] for tty execution: start in background and provide a curses interface. When quit, the background process will keep running
-- [ ] provide a cli for running simple tasks
-- [x] provide a curses mode that displays number of workers, running tasks, remaining tasks, etc. Curses mode is the default.
-- [ ] display ETA in curses mode
-- [ ] curses mode can control (pause/stop/resume) other running executions
+# Fasten your seatbelts! 💺💺💺💺
+
+Run ruby code, shell code, or customized tasks in parallel using worker processes/threads. You can dynamically pause/continue/increase/decrease the number of tasks using our UI.
+
+![](demo/demo.gif)
+
+**WARNING** this is a Work In Progress project:
+- features are still being implemented
+- API could and will change
+
+It has been released so people can try it and make feature requests, comments and suggestions.
+
+## Feature Roadmap
+
+### Task definitions:
+- [x] define tasks from an YAML file
+- [x] tasks definition can include shell code or eval'ed ruby code
+- [x] tasks can also be defined with blocks of code, like RSpec.describe
+- [x] tasks can have dependencies that are needed to reach a goal, similar to rake, make, and others
+
+### Execution:
+- [x] tasks are executed in parallel, using worker processes/threads
+- [x] process are preferred on POSIX systems, otherwise threads are used
+- [x] default number of workers is the # of real cpu cores on the system
+- [x] in case of task Error or Exception, the whole run will be stopped
+- [x] runner saves a log file in `$(pwd)/.fasten/log/runner/*name_of_runner*.log`
+- [x] each task saves its STDERR/STDOUT to a log file in `$(pwd)/.fasten/log/task/*name_of_task*.log`
+- [x] once finished, statistics are saved in `$HOME/.fasten/stats/*name_of_runner*.csv`, which include: status (DONE, FAIL), duration, average, stdev, error and count
+- [x] display a summary table of the execution
+
+### UI/Curses
+- [x] only available if the 'curses' gem is installed
+- [x] Display number of workers running, idle, waiting, and number of workers (max)
+- [x] Display current running tasks, pending tasks, waiting tasks, etc; showing which task is pending because other tasks need to be run first
+- [x] Display the number of tasks done/pending
+- [x] Display a progressbar
+- [x] Display current local time
+- [x] Pause whole runner with `P` key (waits current running tasks end)
+- [x] Resume paused runner with `R` key
+- [x] Press ⬅️ or ➡️ keys to dynamically increase/decrease number of workers
+- [ ] Use ⬆️ and ⬇️ to select tasks
+- [ ] Calculate ETA, assuming all tasks take same time
+- [ ] Calculate ETA, based on saved tasks statistics
+- [ ] Live tail -f of selected tasks
+
+### UI/Console
+- [x] Console is the default if STDOUT/STDIN is not a tty
+- [x] Display each task as is being finished
+- [ ] Calculate ETA, assuming all tasks take same time
+- [ ] Calculate ETA, based on saved tasks statistics
+
+
+### CLI
+- [x] provides a CLI for running *_fasten.rb ruby code
 
 
 ## Installation
@@ -27,6 +60,7 @@ Add this line to your application's Gemfile:
 
 ```ruby
 gem 'fasten'
+gem 'curses' # optional
 ```
 
 And then execute:
@@ -39,7 +73,7 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+TODO: Write usage instructions here. See spec for sample code.
 
 ## Development
 

data/fasten.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|demo)/}) }
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

data/lib/fasten/runner.rb

@@ -20,20 +20,25 @@ module Fasten
     include Fasten::Support::UI
     include Fasten::Support::Yaml
 
-    attr_accessor :name, :workers, :worker_class, :fasten_dir, :developer, :stats, :worker_list, :use_threads, :queue
+    attr_accessor :name, :workers, :worker_class, :fasten_dir, :developer, :stats, :summary, :ui_mode, :worker_list, :use_threads, :queue
 
-    def initialize(name: nil, developer: STDIN.tty? && STDOUT.tty?, workers: Parallel.physical_processor_count, worker_class: Worker, fasten_dir: '.fasten', use_threads: !OS.posix?)
-      reconfigure(name: name, developer: developer, workers: workers, worker_class: worker_class, fasten_dir: fasten_dir, use_threads: use_threads)
+    def initialize(name: nil,
+                   developer: Fasten.default_developer, summary: nil, ui_mode: Fasten.default_ui_mode, workers: Fasten.default_workers,
+                   worker_class: Worker, fasten_dir: '.fasten', use_threads: !OS.posix?)
+      reconfigure(name: name, developer: developer, summary: summary, ui_mode: ui_mode, workers: workers,
+                  worker_class: worker_class, fasten_dir: fasten_dir, use_threads: use_threads)
     end
 
     def reconfigure(**options)
-      self.stats = options[:name] && true if options[:name] || options.key?(:stats)
-      self.name = options[:name] || "#{self.class} #{$PID}" if options[:name]
-      self.workers = options[:workers] if options[:workers]
-      self.worker_class = options[:worker_class] if options[:worker_class]
-      self.fasten_dir = options[:fasten_dir] if options[:fasten_dir]
-      self.developer = options[:developer] if options[:developer]
-      self.use_threads = options[:use_threads] if options[:use_threads]
+      self.stats        = options[:name] && true if options[:name] || options.key?(:stats)
+      self.name         = options[:name] || "#{self.class} #{$PID}" if options.key?(:name)
+      self.workers      = options[:workers]       if options.key?(:workers)
+      self.worker_class = options[:worker_class]  if options.key?(:worker_class)
+      self.fasten_dir   = options[:fasten_dir]    if options.key?(:fasten_dir)
+      self.developer    = options[:developer]     if options.key?(:developer)
+      self.use_threads  = options[:use_threads]   if options.key?(:use_threads)
+      self.summary      = options[:summary]       if options.key?(:summary)
+      self.ui_mode      = options[:ui_mode]       if options.key?(:ui_mode)
 
       initialize_dag
       initialize_stats
@@ -63,6 +68,8 @@ module Fasten
       log_fin self, running_counters
 
       stats_add_entry(state, self)
+
+      stats_summary if summary
     ensure
       save_stats
     end

data/lib/fasten/support/stats.rb

@@ -76,7 +76,7 @@ module Fasten
 
       FLOAT_FORMATTER = ->(f) { format('%7.3f', f) }
 
-      def stats_table_run
+      def stats_summary_data
         sub = stats_entries.select { |x| x['kind'] == 'task' }.map { |x| x['run'] }.sum
         tot = stats_entries.select { |x| x['kind'] == 'runner' }.map { |x| x['run'] }.sum
 
@@ -104,8 +104,8 @@ module Fasten
         str
       end
 
-      def stats_table
-        sub, tot = stats_table_run
+      def stats_summary
+        sub, tot = stats_summary_data
 
         Hirb::Console.render_output(stats_entries,
                                     fields: %w[state kind name run cnt avg std err worker], unicode: true, class: 'Hirb::Helpers::AutoTable',

data/lib/fasten/support/ui.rb

@@ -5,7 +5,12 @@ module Fasten
     module UI
       def ui
         require 'fasten/ui/curses'
-        @ui ||= STDIN.tty? && STDOUT.tty? ? Fasten::UI::Curses.new(runner: self) : Fasten::UI::Console.new(runner: self)
+
+        @ui ||= if ui_mode.to_s == 'curses' && STDIN.tty? && STDOUT.tty?
+                  Fasten::UI::Curses.new(runner: self)
+                else
+                  Fasten::UI::Console.new(runner: self)
+                end
       rescue StandardError, LoadError
         @ui = Fasten::UI::Console.new(runner: self)
       end

data/lib/fasten/ui/curses.rb

@@ -129,7 +129,7 @@ module Fasten
         waiting_count = task_waiting_list.count
         workers_count = worker_list.count
 
-        "Procs run: #{running_count} idle: #{workers_count - running_count} #{runner.use_threads ? 'threads' : 'processes'}: #{workers} wait: #{waiting_count}"
+        "Procs running: #{running_count} idle: #{workers_count - running_count} waiting: #{waiting_count} #{runner.use_threads ? 'threads' : 'processes'}: #{workers}"
       end
 
       def ui_workers

data/lib/fasten/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fasten
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/lib/fasten.rb

@@ -35,6 +35,24 @@ module Fasten
       runner(**options).register(&block)
     end
 
+    def default_workers
+      Parallel.physical_processor_count
+    end
+
+    def default_developer
+      STDIN.tty? && STDOUT.tty?
+    end
+
+    def default_ui_mode
+      return @default_ui_mode if defined? @default_ui_mode
+
+      require 'fasten/ui/curses'
+
+      @default_ui_mode = :curses
+    rescue StandardError, LoadError
+      @default_ui_mode = :console
+    end
+
     def load_fasten(args)
       args.each do |path|
         if File.directory? path
@@ -61,25 +79,33 @@ module Fasten
       @opt_parser = OptionParser.new do |opts|
         opts.banner = "Usage: #{$PROGRAM_NAME} [options] FILE/DIRECTORY"
         opts.separator ''
-        opts.separator 'Options'
+        opts.separator 'Example: fasten --name deploy --summary deploy_script.rb'
+        opts.separator ''
+        opts.separator 'Options:'
 
-        opts.on '-n', '--name NAME', String, "Name of this job" do |name|
+        opts.on '-n NAME', '--name=NAME', String, 'Name of this runner, used to display and save stats' do |name|
           @options[:name] = name
         end
-        opts.on '-w', '--workers WORKERS', Numeric, "Number of processes/threads to use (#{Parallel.physical_processor_count} on this machine)" do |workers|
+        opts.on '-w WORKERS', '--workers=WORKERS', Numeric, "Number of processes/threads to use (default: #{default_workers} on this machine)" do |workers|
           @options[:workers] = workers
         end
-        opts.on '-t', '--threads' do
+        opts.on '-s', '--[no-]summary', TrueClass, 'Display summary at the end of execution' do |boolean|
+          @options[:summary] = boolean
+        end
+        opts.on '--ui=UI', String, "Type of UI: curses, console. (default: #{default_ui_mode} on this machine)" do |ui_mode|
+          @options[:ui_mode] = ui_mode
+        end
+        opts.on '-t', '--threads', 'Use threads workers for parallel execution' do
           @options[:use_threads] = true
         end
-        opts.on '-p', '--processes' do
+        opts.on '-p', '--processes', 'Use process workers for parallel execution' do
           @options[:use_threads] = false
         end
-        opts.on '-v', '--version' do
+        opts.on '-v', '--version', 'Display version info' do
           puts Fasten::VERSION
           exit 0
         end
-        opts.on_tail '-h', '--help' do
+        opts.on_tail '-h', '--help', 'Shows this help' do
           puts opts
           exit 0
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fasten
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Aldrin Martoq
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-06 00:00:00.000000000 Z
+date: 2018-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler