enparallel-bin 1.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a40aa3e43e1fac9e594bbaeadb9786f0899cf34bf0849f73f3c0b9f2d3a01e4f
+  data.tar.gz: 366611b5ea88dc0b621bfd5c65af356a0481a045ccfce94dc41e588e0377be61
+SHA512:
+  metadata.gz: 7ccc4d563a11b51a84049d521db264129a693f78e92f8173ecdd02b02373e184701b8924baa3c2bc20cb853da8765b2156289456fdb714bc87592626642f9403
+  data.tar.gz: 6f019d7df35f9d48225f1b62173c67eb6a65b3ccf04f763d3897de12be2d98312911a9fa11834250290f9c2361cfd89fb94e3013ddfde170342b2f9c7d77aabd

data/bin/enparallel

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/enparallel'
+include Enparallel
+
+class Main
+    def self.run(cli)
+        tasks = cli.inputs.map { |input| Task.new(cli.command, input) }
+        pool = ThreadPool.new(tasks, cli.workers, cli.pick)
+        new(pool).run
+    end
+
+    def initialize(pool)
+        @pool = pool
+        @id = 0
+    end
+
+    def run
+        start
+
+        render_progress do
+            @pool.drain_wait
+        end
+
+        finish
+        save_log
+    end
+
+    private
+
+    def start
+        puts 'Running %d tasks with %d workers' % [@pool.size, @pool.worker_count]
+        puts
+    end
+
+    def finish
+        puts
+        puts "Tasks complete"
+        puts
+        puts "#{@pool.succeeded_tasks.length.to_s.green.bold} tasks succeeded"
+        puts "#{@pool.failed_tasks.length.to_s.red.bold} tasks failed"
+        puts
+    end
+
+    def render_progress
+        thread = Thread.new { loop { render } }
+        yield
+        thread.terminate
+        render
+        puts
+    end
+
+    def render
+        print "\r" + @pool.render
+        sleep 0.1
+    end
+
+    def all_paths_available(paths)
+        paths.all? { |path| !File.exist?(path) }
+    end
+
+    def next_id
+        @id += 1
+    end
+
+    def generate_paths(types)
+        loop do
+            id = next_id
+
+            paths = types.to_h do |type|
+                [type, '/tmp/enparallel-run-%d-%s.txt' % [id, type]]
+            end
+
+            break paths if all_paths_available(paths.values)
+        end
+    end
+
+    def save_log
+        log_groups = @pool.get_log_groups
+        types = log_groups.map(&:type).uniq
+        paths = generate_paths(types)
+
+        log_groups.each do |log_group|
+            type = log_group.type
+            path = paths[type]
+            puts 'Written %s log: %s (%s)' % [type, *log_group.write(path)]
+        end
+    end
+end
+
+begin
+    Main.run(CLI.parse(ARGV, STDIN))
+rescue Docopt::Exit => e
+    puts e.message
+    exit
+end

data/lib/enparallel/cli.rb

@@ -0,0 +1,72 @@
+module Enparallel
+    class CLI
+        def initialize(opts, stdin)
+            @opts = opts
+            @stdin = stdin
+        end
+
+        def self.workers_default
+            Util.processor_count
+        end
+
+        def self.pick_default
+            'sequential'
+        end
+
+        def self.parse(argv, stdin)
+            new(Docopt::docopt(usage, argv: argv, version: VERSION), stdin)
+        end
+
+        def self.basename
+            File.basename($0)
+        end
+
+        def self.usage
+            <<~EOF
+                #{'Usage:'.bold}
+                    #{basename} [options] [--] <command>...
+
+                #{'Description:'.bold}
+                    #{basename} operates by reading lines from standard input, and executing
+                    <command> once per entry, in parallel.
+
+                    The placeholder "{}", if present, is replaced with each line of input in turn.
+
+                    seq 1 10 | enparallel sleep {}
+
+                    To run a more complex command or to make use of shell functions or constructs
+                    (enparallel runs its argument as a program) use a call to "bash -c". Note that
+                    because of the "-c" you need to prefix the command with "--" to indicate the
+                    end of parameters to enparallel.
+
+                    seq 1 10 | enparallel -- bash -c "sleep {} && echo Slept for {}"
+
+                #{'Options:'.bold}
+                    -w, --workers <n>   Batch into a pool of <n> workers [default: #{workers_default}].
+                    -p, --pick <type>   Task-picking rule (see "Types") [default: #{pick_default}].
+                    -v, --version       Version.
+                    -h, --help          Help.
+
+                #{'Types:'.bold}
+                    sequential          The order in which the tasks were queued.
+                    random              Random order.
+            EOF
+        end
+
+        def inputs
+            @inputs ||= @stdin.each_line.map(&:chomp)
+        end
+
+        def command
+            Command.from_a(@opts['<command>'])
+        end
+
+        def workers
+            @opts['--workers'].to_i || workers_default
+        end
+
+        def pick
+            @opts['--pick'] || pick_default
+        end
+    end
+end

data/lib/enparallel/command.rb

@@ -0,0 +1,27 @@
+module Enparallel
+    class Command
+        def initialize(name, args)
+            @name = name
+            @args = args
+        end
+
+        def self.from_a(a)
+            name, *args = a
+            Command.new(name, args)
+        end
+
+        def interpolate_safe(replacement)
+            [@name, *replace(replacement)].shelljoin
+        end
+
+        def interpolate_unsafe(replacement)
+            [@name, *replace(replacement)].join(' ')
+        end
+
+        private
+
+        def replace(replacement)
+            @args.map { |arg| arg.gsub('{}', replacement) }
+        end
+    end
+end

data/lib/enparallel/log_group.rb

@@ -0,0 +1,35 @@
+module Enparallel
+    class LogGroup
+        attr_reader :type, :tasks
+
+        def initialize(type, tasks)
+            @type = type
+            @tasks = tasks
+        end
+
+        def to_soml
+            tasks.join("\n\n") + "\n"
+        end
+
+        def has_tasks?
+            tasks.length > 0
+        end
+
+        def write(path)
+            size = File.write(path, to_soml)
+            [path, Util.bytes_to_human(size)]
+        end
+
+        def self.of(type, pool)
+            LogGroup.new(type, pool.tasks_of(type))
+        end
+
+        def self.success(pool)
+            of(:success, pool)
+        end
+
+        def self.failure(pool)
+            of(:failure, pool)
+        end
+    end
+end

data/lib/enparallel/logger.rb

@@ -0,0 +1,19 @@
+module Enparallel
+    class Logger
+        def self.from_thread_pool(pool)
+            Logger.new(pool)
+        end
+
+        def initialize(pool)
+            @pool = pool
+        end
+
+        def get_all_log_groups
+            [LogGroup.success(@pool), LogGroup.failure(@pool)]
+        end
+
+        def get_log_groups
+            get_all_log_groups.select(&:has_tasks?)
+        end
+    end
+end

data/lib/enparallel/picker.rb

@@ -0,0 +1,20 @@
+module Enparallel
+    class Picker
+        def initialize(items, rule)
+            if rule == :random
+                items = items.shuffle
+            end
+
+            @items = items
+            @i = -1
+        end
+
+        def next
+            @items[next_index]
+        end
+
+        def next_index
+            @i += 1
+        end
+    end
+end

data/lib/enparallel/task.rb

@@ -0,0 +1,71 @@
+module Enparallel
+    class Task
+        attr_accessor :stdout
+        attr_accessor :stderr
+
+        def initialize(command, input)
+            @command = command
+            @input = input
+            @running = false
+            @stdout = ''
+            @stderr = ''
+            @ran_at = nil
+            @exit_status = nil
+        end
+
+        def to_s
+            document = SOML::Document.new
+
+            document.add('CommandLine', command_line_unsafe)
+            document.add('ExitStatus', @exit_status)
+            document.add('RanAt', @ran_at)
+            document.add('StandardOutput', @stdout) unless @stdout.empty?
+            document.add('StandardError', @stderr) unless @stderr.empty?
+
+            document.to_s
+        end
+
+        def char
+            if @running
+                'R'
+            elsif @exit_status.nil?
+                'S'
+            elsif has_succeeded?
+                'D'.green
+            else
+                'F'.red
+            end
+        end
+
+        def run
+            @running = true
+            @ran_at = Time.now
+
+            Open3.popen3(command_line_safe) do |stdin, stdout, stderr, thread|
+                @stdout = stdout.read.chomp
+                @stderr = stderr.read.chomp
+                @exit_status = thread.value.exitstatus
+            end
+        rescue => e
+            @stderr = e.message
+            @exit_status = 1
+        ensure
+            @running = false
+        end
+
+        def has_succeeded?
+            raise 'Task not resolved' if @ran_at.nil?
+            @exit_status == 0
+        end
+
+        private
+
+        def command_line_safe
+            @command.interpolate_safe(@input)
+        end
+
+        def command_line_unsafe
+            @command.interpolate_unsafe(@input)
+        end
+    end
+end

data/lib/enparallel/thread_pool.rb

@@ -0,0 +1,64 @@
+module Enparallel
+    class ThreadPool
+        attr_accessor :worker_count
+
+        def initialize(tasks, requested_worker_count, rule)
+            @tasks = tasks
+            @worker_count = [tasks.length, requested_worker_count].min
+            @picker = Picker.new(tasks, rule.to_sym)
+        end
+
+        def drain_wait
+            drain
+            join
+        end
+
+        def drain
+            @workers = @worker_count.times.map do
+                Thread.new do
+                    while task = @picker.next
+                        task.run
+                    end
+                end
+            end
+        end
+
+        def size
+            @tasks.length
+        end
+
+        def render
+            @tasks.map(&:char).join.bold
+        end
+
+        def succeeded_tasks
+            @tasks.select(&:has_succeeded?)
+        end
+
+        def failed_tasks
+            @tasks.reject(&:has_succeeded?)
+        end
+
+        def tasks_of(type)
+            if type == :success
+                succeeded_tasks
+            elsif type == :failure
+                failed_tasks
+            end
+        end
+
+        def get_log_groups
+            logger.get_log_groups
+        end
+
+        def logger
+            @logger ||= Logger.from_thread_pool(self)
+        end
+
+        private
+
+        def join
+            @workers.each(&:join)
+        end
+    end
+end

data/lib/enparallel/util.rb

@@ -0,0 +1,15 @@
+module Enparallel
+    class Util
+        def self.processor_count
+            Etc.nprocessors
+        end
+
+        def self.bytes_to_human(size)
+            if size >= 1024
+                '%sK' % (size / 1024)
+            else
+                '%sB' % size
+            end
+        end
+    end
+end

data/lib/enparallel/version.rb

@@ -0,0 +1,3 @@
+module Enparallel
+    VERSION = "1.0.4"
+end

data/lib/enparallel.rb

@@ -0,0 +1,12 @@
+require 'require_all'
+require 'colorize'
+require 'docopt'
+require 'soml'
+
+require 'securerandom'
+require 'shellwords'
+require 'tempfile'
+require 'open3'
+require 'etc'
+
+require_rel 'enparallel'

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: enparallel-bin
+version: !ruby/object:Gem::Version
+  version: 1.0.4
+platform: ruby
+authors:
+- crdx
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: require_all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: docopt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.1
+- !ruby/object:Gem::Dependency
+  name: soml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+description: 
+email: 
+executables:
+- enparallel
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/enparallel
+- lib/enparallel.rb
+- lib/enparallel/cli.rb
+- lib/enparallel/command.rb
+- lib/enparallel/log_group.rb
+- lib/enparallel/logger.rb
+- lib/enparallel/picker.rb
+- lib/enparallel/task.rb
+- lib/enparallel/thread_pool.rb
+- lib/enparallel/util.rb
+- lib/enparallel/version.rb
+homepage: https://github.com/crdx/enparallel
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Run many commands enparallel with a colourful overview
+test_files: []