peer_commander 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e49ccbfcf6d4da8b75b671e3cd4b9c2f36ac08fd
+  data.tar.gz: ba81b657c8d86be0551449a50e8fd63068990a8d
+SHA512:
+  metadata.gz: 486680f237079ba157c1756a057125b6af768a9607be30ed947762a5e53b3c2306cdc174c4d90c273e759adf529d772430aa1beb12f2ddd1500e6d079f5b395a
+  data.tar.gz: b2744de1e0247d9a5b8ff0eda42784dcbcf5336d4046ef9593d336e51ffa39edb95774ba7a98724aa689aa47ddd0b66c13be150c386428ef5516b71e3a9d1cfa

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+Gemfile.lock
+
+peer-commander-*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,30 @@
+AllCops:
+  TargetRubyVersion: 2.4
+
+Metrics/LineLength:
+  Max: 119
+  Exclude:
+    - "spec/**/*"
+
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/LetSetup:
+  Enabled: false
+
+Style/DotPosition:
+  EnforcedStyle: trailing
+
+Style/SingleLineBlockParams:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+  Enabled: true
+
+Style/FrozenStringLiteralComment:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Jonathan Harden
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,94 @@
+# Peer::Commander
+
+Provides an interfacing for executing mutliple system commands with a configurable level of parallelism.
+
+The result objects will have access to the output of the command as well as the return status.
+
+The overall result of executing the commands will be successful if all of the commands executed completed with
+a successful return code.
+
+Internally commands are executed with Open3.capture2e, env, opts, and stdin\_data used to init the commands are
+sent directly to Open3.capture2e, see the documentation for that class to understand the behaviour and options
+available.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "peer_commander"
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install peer_commander
+
+## Usage
+
+This is a convoluted example to demonstrate all the features
+
+```ruby
+require "peer_commander"
+
+commands = [
+  PeerCommander::Command.new("sleep 2"),
+  PeerCommander::Command.new("wibble"),
+  PeerCommander::Command.new("sleep 1"),
+  PeerCommander::Command.new("exit 1"),
+  PeerCommander::Command.new("echo $TEST_ENV", env: { "TEST_ENV" => "output from env var"}),
+  PeerCommander::Command.new("cat", stdin_data: "data passed to standard in"),
+  PeerCommander::Command.new("sleep 4", opts: { chdir: "/tmp" }), # This command will be executed in the directory /tmp
+]
+
+commander = PeerCommander::CommandRunner.new(commands)
+
+results = commander.execute(parallelism: 4)
+
+puts "SUCCESS" if commander.success?
+puts "FAILED" if commander.failed?
+
+commander.all_commands.each { |command| puts "Executed #{command.command}" }
+
+commander.successful_commands.each do |command|
+  puts "Command [#{command.command}] succeeded in #{command.duration} seconds with output:"
+  puts command.output
+  puts "================================================="
+end
+
+commander.failed_commands.each do |command|
+  print "Command [#{command.command}] failed in #{command.duration} seconds "
+
+  if command.exception
+    puts "with exception #{command.exception}"
+  else
+    puts "with exit code #{command.exit_code} and output\n"
+    puts command.output
+  end
+
+  puts "================================================="
+end
+
+puts "Total operation took #{commander.duration} seconds"
+```
+
+## Version numbering
+
+This project will follow strict semantic versioning (https://semver.org/spec/v2.0.0.html) once we reach version 0.1.
+Until then any version change can break anything.
+
+## Development
+
+You can run `bin/rspec` or `bin/rake` to run all the tests, you can also run `bin/console` for an interactive prompt
+that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on Bitbucket at https://bitbucket.org/eviljonny/peer_commander/
+
+## License
+
+This project is released under the MIT license which can be found in the LICENSE file

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require_relative "../lib/peer_commander"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 150) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 150) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/lib/peer_commander.rb

@@ -0,0 +1,3 @@
+require_relative "peer_commander/errors"
+require_relative "peer_commander/command_runner"
+require_relative "peer_commander/version"

data/lib/peer_commander/command.rb

@@ -0,0 +1,92 @@
+require "open3"
+
+module PeerCommander
+  # A single command to be executed, with access to results statuses, output, and timings
+  class Command
+    attr_reader :command, :exception, :opts, :env, :stdin_data
+
+    # The env specified will be passed directly to Open3.capture2e as the env to run with
+    # The opts provided will also be passed directly to Open3.capture2e as opts
+    # stdin_data will also be passed directly to Open3.capture2e
+    def initialize(command, stdin_data: nil, env: {}, opts: {})
+      @command = command
+      @stdin_data = stdin_data
+      @env = env
+      @opts = opts
+    end
+
+    # Execute the command, will capture any exceptions that inherit from StandardError raised by the execution of
+    # the given command and store it in the exception attribute, NOTE: This means it _will not_ propogate the
+    # exception upwards
+    #
+    # Returns self
+    def execute
+      raise Errors::CommandAlreadyExecutedError if executed?
+
+      start_time = Time.now
+
+      begin
+        @command_output, @status = Open3.capture2e(env, command, execution_options)
+      rescue StandardError => e
+        @exception = e
+      ensure
+        @timing = Time.now - start_time
+      end
+
+      self
+    end
+
+    # Return true if the command was successful
+    def success?
+      raise Errors::CommandNotExecutedError unless executed?
+
+      exception.nil? && status.success?
+    end
+
+    # Return true if the command failed
+    def failed?
+      !success?
+    end
+
+    # Return the output (stdout and stderr interleaved) of running the command
+    def output
+      raise Errors::CommandNotExecutedError unless executed?
+
+      command_output
+    end
+
+    # How long the command took to run in seconds
+    def duration
+      raise Errors::CommandNotExecutedError unless executed?
+
+      timing
+    end
+
+    # Returns a truthy object if the command has been executed, or nil
+    # if it has not.
+    #
+    # If the command has been executed it will either return the status code or
+    # the exception that was raised
+    def executed?
+      exception || status
+    end
+
+    def exit_code
+      raise Errors::CommandNotExecutedError unless executed?
+
+      return nil if exception
+
+      status.exitstatus
+    end
+
+    private
+
+    attr_reader :status, :command_output, :timing
+
+    def execution_options
+      opts.tap do |options|
+        options[:stdin_data] = stdin_data unless stdin_data.nil?
+      end
+    end
+  end
+end

data/lib/peer_commander/command_runner.rb

@@ -0,0 +1,57 @@
+require_relative "command"
+require_relative "parallel_executor"
+
+module PeerCommander
+  # Runs the listed commands with a configurable level of parallelism
+  class CommandRunner
+    def initialize(commands)
+      @commands = commands
+      @command_results = []
+    end
+
+    def execute(parallelism: 1)
+      start = Time.now
+      @command_results = ParallelExecutor.new.execute(commands, parallelism)
+      @duration = Time.now - start
+      @command_results
+    end
+
+    def success?
+      raise Errors::CommandNotExecutedError if command_results.empty?
+
+      @command_results.all?(&:success?)
+    end
+
+    def failed?
+      !success?
+    end
+
+    def all_commands
+      raise Errors::CommandNotExecutedError if command_results.empty?
+
+      @command_results
+    end
+
+    def successful_commands
+      raise Errors::CommandNotExecutedError if command_results.empty?
+
+      @command_results.select(&:success?)
+    end
+
+    def failed_commands
+      raise Errors::CommandNotExecutedError if command_results.empty?
+
+      @command_results.select(&:failed?)
+    end
+
+    def duration
+      raise Errors::CommandNotExecutedError if command_results.empty?
+
+      @duration
+    end
+
+    private
+
+    attr_reader :commands, :command_results
+  end
+end

data/lib/peer_commander/errors.rb

@@ -0,0 +1,2 @@
+require_relative "errors/command_already_executed_error"
+require_relative "errors/command_not_executed_error"

data/lib/peer_commander/errors/command_already_executed_error.rb

@@ -0,0 +1,5 @@
+module PeerCommander
+  module Errors
+    CommandAlreadyExecutedError = Class.new(StandardError)
+  end
+end

data/lib/peer_commander/errors/command_not_executed_error.rb

@@ -0,0 +1,5 @@
+module PeerCommander
+  module Errors
+    CommandNotExecutedError = Class.new(StandardError)
+  end
+end

data/lib/peer_commander/parallel_executor.rb

@@ -0,0 +1,63 @@
+require "concurrent"
+
+module PeerCommander
+  # Executes a given set of commands with a specified parallelism
+  class ParallelExecutor
+    SLEEP_DURATION = 0.2
+
+    def initialize
+      @futures = []
+      @command_results = []
+    end
+
+    def execute(commands, parallelism)
+      raise ArgumentError, "Parallelism must be at least 1" if parallelism < 1
+
+      @parallelism = parallelism
+
+      commands.each do |command|
+        wait_for_slot if all_slots_filled?
+
+        @futures << Concurrent::Future.execute { command.execute }
+      end
+
+      wait_for_all
+
+      command_results
+    end
+
+    private
+
+    attr_reader :parallelism, :futures, :command_results, :future_command_map
+
+    def wait_for_slot
+      while all_slots_filled?
+        sleep(SLEEP_DURATION)
+        remove_completed_commands
+      end
+    end
+
+    def wait_for_all
+      sleep(SLEEP_DURATION) while futures.any?(&:incomplete?)
+      @command_results.push(*extract_results_from(futures))
+    end
+
+    def remove_completed_commands
+      completed = futures.select(&:complete?)
+      @futures -= completed
+      @command_results.push(*extract_results_from(completed))
+    end
+
+    def slot_available?
+      futures.size < parallelism
+    end
+
+    def all_slots_filled?
+      !slot_available?
+    end
+
+    def extract_results_from(futures_to_extract)
+      futures_to_extract.map(&:value)
+    end
+  end
+end

data/lib/peer_commander/version.rb

@@ -0,0 +1,3 @@
+module PeerCommander
+  VERSION = "0.1.0"
+end

data/peer_commander.gemspec

@@ -0,0 +1,30 @@
+
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require_relative "lib/peer_commander/version"
+
+Gem::Specification.new do |spec|
+  spec.name                  = "peer_commander"
+  spec.version               = PeerCommander::VERSION
+  spec.authors               = ["Jonathan Harden"]
+  spec.email                 = ["jonathan.harden@mydrivesolutions.com"]
+  spec.required_ruby_version = ">= 2.4"
+
+  spec.licenses              = ["MIT"]
+
+  spec.summary               = "Run arbitrary system commands in parallel reporting on errors."
+  spec.homepage              = "https://bitbucket.org/eviljonny/peer_commander/"
+
+  spec.files                 = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "concurrent-ruby", "~> 1"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "pry-byebug", "~> 3"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: peer_commander
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jonathan Harden
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email:
+- jonathan.harden@mydrivesolutions.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/rake
+- bin/rspec
+- lib/peer_commander.rb
+- lib/peer_commander/command.rb
+- lib/peer_commander/command_runner.rb
+- lib/peer_commander/errors.rb
+- lib/peer_commander/errors/command_already_executed_error.rb
+- lib/peer_commander/errors/command_not_executed_error.rb
+- lib/peer_commander/parallel_executor.rb
+- lib/peer_commander/version.rb
+- peer_commander.gemspec
+homepage: https://bitbucket.org/eviljonny/peer_commander/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: Run arbitrary system commands in parallel reporting on errors.
+test_files: []