parenting 0.1.5

data/README.md

@@ -0,0 +1,49 @@
+# Parenting
+
+Sometimes you have a lot of stuff to do, and you want to get it done fast.
+What's the obvious thing to do? Have a bunch of kids, and tell *them* to do it, of course!
+
+## Intent
+
+This gem allows a straightforward model of multiprocessing: spin off multiple external programs
+to do work, and then wait for them to finish. That use-case is already possible in a basic
+way using `spawn`, but that gives you no control or interactivity with the spawned process.
+
+Because you may have more tasks than can reasonably run simultaneously (memory restrictions,
+processor count, etc), you usually would like to specify an upper-limit on how many of those
+tasks may be running simultaneously. If you have chores that are disproportionately long, you
+usually want to minimize the net run-time - Parenting allows you to specify a 'cost' for each chore,
+which it will use to sort them into longest-job first (provably minimzing the net run-time).
+
+The most important detail is that your chores will probably want to do some kind of logging,
+so that your main process can tell what is going on in each of them. The natural way to do this
+is to allow the external processes to log via stderr, and to do something with each line of log
+so produced - each chore takes a callable for how to thread-safely handle that output.
+
+You can pass input to the child process via the `:stdin` option, but it is not intended for
+bulk interaction - that string is fed to the process immediately, and the pipe is then closed.
+
+## Thread-Safety
+
+Parenting uses threads internally to allow jobs to finish in arbitrary order. None of the callbacks
+you initialize chores with will be used outside of the main thread however, and all data structures
+passed into the options hash will be dup'd, so you can safely reuse them for multiple chores.
+
+## Usage
+
+```ruby
+
+# build a coordinator that allows 4 children at a time
+boss = Parenting::Boss.new(4)
+
+['ls', 'ls -l', 'ls -a', 'echo hello'].each do |cmd|
+  boss.add_chore({
+    :command    => cmd,
+    :on_success => lambda { STDERR.puts "#{cmd} succeeded" },
+    :on_failure => lambda { STDERR.puts "#{cmd} failed"    },
+    :on_stderr  => lambda { |ln| STDERR.puts "#{cmd} produced: #{ln}" }
+  })
+end
+
+boss.run!
+```

data/lib/parenting/boss.rb

@@ -0,0 +1,93 @@
+require 'set'
+
+module Parenting
+  class Boss
+    attr_accessor :max_children, :chores, :in_progress, :completed
+
+    def initialize(number_of_children)
+      self.max_children = number_of_children
+      self.chores = []
+      self.in_progress = []
+      self.completed = Set.new
+    end
+
+    def add_chore(opts)
+      self.chores << Parenting::Chore.new(opts)
+    end
+
+    def assign_next_chore
+      return unless self.assignable_chores.any?
+      return if self.in_progress.length >= self.max_children
+
+      next_location = self.chores.find_index{|c| c.satisfied? self.completed }
+      next_chore = self.chores.delete_at next_location
+      next_chore.run!
+
+      self.in_progress << next_chore
+    end
+
+    def free_children?
+      self.in_progress.length < self.max_children
+    end
+
+    def done?
+      self.assignable_chores.empty? && self.in_progress.empty?
+    end
+
+    def handle_complaints
+      self.in_progress.each do |chore|
+        until chore.stderr.empty?
+          chore.on_stderr.call(chore.stderr.shift)
+        end
+      end
+    end
+
+    def check_children
+      remaining = []
+      self.in_progress.each do |chore|
+        if chore.complete?
+          chore.handle_completion
+          self.completed << chore.name if chore.name and not chore.failed?
+        else
+          remaining << chore
+        end
+      end
+
+      self.in_progress = remaining
+
+      self.in_progress.each do |chore|
+        until chore.completed.empty?
+          self.completed << chore.shift
+        end
+      end
+    end
+
+    def assignable_chores
+      self.chores.select do |c|
+        c.satisfied? self.completed
+      end
+    end
+
+    def run!
+      # get the chores into longest job first - this is to minimize net runtime
+      self.chores = self.chores.sort_by{|c| - c.cost.to_f}
+
+      # queue up the first set of chores
+      while self.assignable_chores.any? and self.free_children?
+        self.assign_next_chore
+      end
+
+      # watch the children, and assign new chores if any get free
+      until self.done?
+        sleep 0.05
+
+        self.handle_complaints
+        self.check_children
+
+        while self.free_children? and self.assignable_chores.any?
+          self.assign_next_chore
+        end
+      end
+    end
+  end
+end

data/lib/parenting/chore.rb

@@ -0,0 +1,78 @@
+module Parenting
+  class Chore
+    attr_accessor :on_success, :on_failure, :on_stderr, :exit_status
+    attr_accessor :command, :stdin, :stdout, :stderr
+    attr_accessor :cost, :thread, :result
+    attr_accessor :deps, :name, :completed
+
+    def initialize(opts)
+      [:on_success, :on_failure, :on_stderr].each do |cb|
+        self.send :"#{cb}=", opts.fetch(cb).dup
+      end
+
+      self.name = opts[:name] || nil
+      self.deps = opts[:deps] || []
+      self.completed = Queue.new
+
+      self.command = opts.fetch(:command).dup
+      self.command = [self.command] unless self.command.is_a? Array
+      self.cost    = opts[:cost] || nil
+      self.stdin   = opts[:stdin] || nil
+      self.stdout  = nil
+      self.stderr  = Queue.new
+      self.result  = :working
+    end
+
+    def satisfied?(completed)
+      self.deps.empty? || self.deps.all?{|d| completed.include?(d)}
+    end
+
+    def run!
+      self.thread = Thread.new do
+        cmd = [self.command].flatten
+        Open3.popen3(* cmd) do |i, o, e, t|
+          i.write(self.stdin); i.close
+
+          e.each_line do |line|
+            self.stderr << line
+          end
+          e.close
+
+          self.stdout = o.read
+          o.close
+
+          result = t.value
+          self.exit_status = result.exitstatus
+
+          if result.success?
+            self.result = :success
+          else
+            self.result = :failure
+          end
+        end
+      end
+    end
+
+    def complete?
+      self.result == :success || self.result == :failure
+    end
+
+    def done_with(name)
+      self.completed << name
+    end
+
+    def handle_completion
+      if self.result == :success
+        self.on_success.call(self)
+      elsif self.result == :failure
+        self.on_failure.call(self)
+      else
+        raise "This should not happen"
+      end
+    end
+
+    def failed?
+      self.result == :failure
+    end
+  end
+end

data/lib/parenting/version.rb

@@ -0,0 +1,3 @@
+module Parenting
+  VERSION = "0.1.5"
+end

data/lib/parenting.rb

@@ -0,0 +1,9 @@
+require 'open3'
+require 'thread'
+require 'io/wait'
+require_relative './parenting/boss'
+require_relative './parenting/chore'
+require_relative './parenting/version'
+
+module Parenting
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: parenting
+version: !ruby/object:Gem::Version
+  version: 0.1.5
+  prerelease: 
+platform: ruby
+authors:
+- Eric Mueller
+autorequire: 
+bindir: scripts
+cert_chain: []
+date: 2013-08-23 00:00:00.000000000 Z
+dependencies: []
+description: Manage multiple child-processes via green threads
+email: emueller@emcien.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/parenting.rb
+- lib/parenting/chore.rb
+- lib/parenting/boss.rb
+- lib/parenting/version.rb
+homepage: http://github.com/emcien/parenting
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Put those child-processes to WORK
+test_files: []