job_pool 0.5 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a9c845f04a84fb0ae22818bec9b471bac197308
-  data.tar.gz: bdadd6aea6d4c068821a24d0afeb316d3f481f8b
+  metadata.gz: 715c70888317b1db49c5644d30e1197d6cd3dcb6
+  data.tar.gz: 15094b1d5774c55b9bb21d469405c65f7ea28942
 SHA512:
-  metadata.gz: 9fbbb0d9a5ab5bc2c811154b2313a5ec834aef8efca4af9bb17b4cd8d75f17b6964a6170d34418e433efa3851f1aed12c4e5ada2404355c3aab3b3c0301f4a07
-  data.tar.gz: 40f58a16103ca5f5722995ee640ce7b9740b7ceba2d22f027cc3f9fb5eb7832d39255b972196497558a460954794d3e48e300c46a0e81e538beb518181df9cd2
+  metadata.gz: 239d6296f2933a8e63a0747b99f1967720b1bb4137c2322b2bf3b90f0c3e07f521a4822e5ebeddd8e942a66d6bcc7f2e2bd00d62f62a68d3a4f9de588b426109
+  data.tar.gz: 04597306e45e9b2d83538bb1a1d4585ecb8a00f48229a6aeb5df2db7082c6de3645581441d2931941593d2f98633cf0d15b180120ee679aaae7b5d0511b745f9

data/.gitignore

@@ -1 +1,3 @@
-Gemfile.lock
+/Gemfile.lock
+/doc
+/.yardoc

data/README.md

@@ -35,7 +35,7 @@ pool = JobPool.new
 Then fire off a job.  This one waits a bit and then ROT-13s its input.
 
 ```ruby
-job = pool.launch("sleep 5; tr A-Za-z N-ZA-Mn-za-m", "the secrets")
+job = pool.launch("sleep 5; tr A-Za-z N-ZA-Mn-za-m", stdin: "the secrets")
 pool.count       => 1
 job.output       => ""
            (after five seconds)
@@ -47,14 +47,16 @@ job.output       => "gur frpergf"
 
 You can specify IO objects to read from and write to:
 
-TODO: this works, but it closes your stdout!  That's problematic.
-Maybe add a mode that doesn't close the output stream when done?
-Or just use a different example?
-
 ```ruby
-pool.launch 'gunzip --to-stdout', File.open('contents.txt.gz'), STDOUT
+source = File.open('contents.txt.gz')
+desdtination = File.open('/tmp/out', 'w')
+pool.launch 'gunzip --to-stdout', stdin: source, stdout: destination
 ```
 
+Note that if you specify STDIN or STDOUT, job_pool will close the stream
+when the child terminates.  This is almost certainly not what you want.
+TODO: add an option... `keep_open: :stdout`, `keep_open: [:stdin, :stdout]`
+
 #### Killing a Job
 
 If you want to terminate a job, just kill it:
@@ -73,7 +75,7 @@ Pass the number of seconds to wait, default is 2 seconds.
 
 #### Timeouts
 
-TODO
+TODO: add a timeout example
 
 #### Limiting Running Processes
 
@@ -103,6 +105,15 @@ TODO: friggin documentation!
 TODO: include an example of a job queue
 
 
+## Documentation
+
+I tried to use Ruby's automated documentation tools but it didn't stick.
+RDoc didn't have reliable markdown support and lacked param name checking.
+YARD produces uglier output (especially noframes) and its lack of :nodoc:
+makes my docs too noisy.  For now I'll just write documentation how I want
+to see it and hope some tool catches up.
+
+
 ## License
 
 MIT, enjoy!

data/Rakefile

@@ -1,6 +1,14 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+require 'yard'
 
 RSpec::Core::RakeTask.new(:spec)
 task :default => ['spec']
 task :test => ['spec']
+
+YARD::Rake::YardocTask.new do |t|
+  t.files   = ['lib/**/*.rb']
+  # t.options = ['--any', '--extra', '--opts']
+  t.stats_options = ['--list-undoc']
+end
+task :doc => ['yard']

data/lib/job_pool.rb

@@ -6,10 +6,13 @@ require 'job_pool/job'
 # TODO: rewrite wait_next
 
 class JobPool
+  # Indicates that the maximum allowed jobs are already running.  Set in [initialize], thrown by [launch].
   class TooManyJobsError < StandardError; end
 
   attr_accessor :max_jobs
 
+  # ## Options
+  # * max_jobs: the maximum number of jobs that can be running at any one time, or nil if unlimited.
   def initialize(options={})
     @mutex ||= Mutex.new
 

data/lib/job_pool/job.rb

@@ -11,36 +11,64 @@ class JobPool; end
 # A job keeps track of the child process that gets forked.
 # job is the Ruby data structure, process is the Unix process.
 class JobPool::Job
-  attr_reader :start_time, :stop_time  # start and finish times of this job
-  attr_reader :inio, :outio, :errio    # fds for child's stdin/stdout/stderr
-
-  # runs cmd, passes instr on its stdin, and fills outio and
-  # errio with the command's output.
-  # TODO: should specify args using keywords rather than position.
-  def initialize pool, cmd, inio=nil, outio=nil, errio=nil, timeout=nil
+  attr_reader :start_time, :stop_time     # start and finish times of this job
+  attr_reader :stdin, :stdout, :stderr    # fds for child's stdin/stdout/stderr
+
+  # **internal**: Use [JobPool#launch], don't call this method directly.
+  #
+  # Starts a process.
+  #
+  # ## Parameters
+  #
+  # * pool [JobPool]: The pool that will contain this job.
+  # * command [String, Array]: The command to run.  Can be specified either
+  #                  as a string or an array of arguments for Process.spawn.
+  #
+  # ## Options
+  #
+  # * stdin [IO, String]: The child's input.  If an IO object isn't supplied,
+  #        an IOString will be created by calling the parameter's to_s method.
+  # * stdout [IO]: the IO object to receive the child's output.
+  # * stderr [IO]: the IO object to receive the child's stderr.
+  # * timeout [seconds]: the number of seconds to wait before killing the job.
+  #
+  # If `stdin`, `stdout`, or `stderr` are omitted, an empty IOString will be created.
+  # If output and error are IOStrings, the [output] method will return the child's
+  # stdout, and [error] will return its stderr.
+  #
+  # ## Examples
+  #
+  # * Simple invocation:            `job = Job.new pool, 'echo hi'`
+  # * Redirect outpout to a file:   `Job.new pool, 'wkhtmltopdf', stdout: File.new('/tmp/out.pdf', 'w')`
+  # * Passing an array and options: `Job.new pool, ['cat', '/tmp/infile', {pgroup: true}]`
+
+  def initialize pool, command, options={}
     @start_time = Time.now
-    @pool  = pool
-    @inio  = inio || StringIO.new
-    @inio  = StringIO.new(@inio.to_s) unless @inio.respond_to?(:readpartial)
-    @outio = outio || StringIO.new
-    @errio = errio || StringIO.new
-    @chin, @chout, @cherr, @child = Open3.popen3(*cmd)
+    @pool   = pool
+    @killed = false
+    @timed_out = false
 
-    @pool._add(self)
+    @stdin  = options[:stdin] || StringIO.new
+    @stdin  = StringIO.new(@stdin.to_s) unless @stdin.respond_to?(:readpartial)
+    @stdout = options[:stdout] || StringIO.new
+    @stderr  = options[:stderr] || StringIO.new
+
+    @chin, @chout, @cherr, @child = Open3.popen3(*command)
     @chout.binmode
 
-    @killed = false
-    @timed_out = false
+    @pool._add(self)
 
-    @thrin  = Thread.new { drain(@inio, @chin) }
-    @throut = Thread.new { drain(@chout, @outio) }
-    @threrr = Thread.new { drain(@cherr, @errio) }
+    @thrin  = Thread.new { drain(@stdin, @chin) }
+    @throut = Thread.new { drain(@chout, @stdout) }
+    @threrr = Thread.new { drain(@cherr, @stderr) }
 
     # ensure cleanup is called when the child exits. (crazy that this requires a whole new thread!)
     @cleanup_thread = Thread.new do
-      if timeout
-        # TODO: inline outatime
-        outatime unless @child.join(timeout)
+      if options[:timeout]
+        unless @child.join(timeout)
+          @timed_out = true
+          kill
+        end
       else
         @child.join
       end
@@ -48,20 +76,25 @@ class JobPool::Job
     end
   end
 
+  # @param [Hash] opts the options to create a message with.
+  # @option opts [String] :subject The subject
+  # @option opts [String] :from ('nobody') From address
+  # @option opts [String] :to Recipient email
+  # @option opts [String] :body ('') The email's body
   def write *args
-    @inio.write *args
+    @stdin.write *args
   end
 
   def read *args
-    @outio.read *args
+    @stdout.read *args
   end
 
   def output
-    @outio.string
+    @stdout.string
   end
 
   def error
-    @errio.string
+    @stderr.string
   end
 
   def finished?
@@ -131,11 +164,6 @@ private
     @cleanup_thread.join unless Thread.current == @cleanup_thread
   end
 
-  def outatime
-    @timed_out = true
-    kill
-  end
-
   # reads every last drop, then closes both files.  must be threadsafe.
   def drain reader, writer
     begin

data/lib/job_pool/version.rb

@@ -1,3 +1,3 @@
 class JobPool
-  VERSION = "0.5"
+  VERSION = "0.6" # :nodoc:
 end

data/spec/job_pool/job_spec.rb

@@ -5,7 +5,7 @@ require 'job_pool/job'
 #    while rspec spec/job_pool/job_spec.rb ; do : ; done
 
 describe JobPool::Job do
-  class FakeJobPool
+  class FakeJobPool # :nodoc:  massively oversimplified job pool used in some testing
     def initialize
       @jobs = []
     end
@@ -24,11 +24,9 @@ describe JobPool::Job do
   end
 
   let(:pool)  { FakeJobPool.new }
-  let(:chin)  { StringIO.new('small instring') }
-  let(:chout) { StringIO.new }
-  let(:cherr) { StringIO.new }
+  let(:small_input)  { StringIO.new('small instring') }
 
-  def time_this_block &block
+  def time_this_block &block # :nodoc:
     start = Time.now
     block.call
     finish = Time.now
@@ -38,9 +36,9 @@ describe JobPool::Job do
 
   it "has a working drain method" do
     bigin = StringIO.new('x' * 1024 * 1024) # at least 1 MB of data to test drain loop
-    job = JobPool::Job.new(pool, 'cat', bigin, chout, cherr)
+    job = JobPool::Job.new(pool, 'cat', stdin: bigin)
     job.stop
-    expect(chout.string).to eq bigin.string
+    expect(job.output).to eq bigin.string
     expect(job.finished?).to eq true
   end
 
@@ -48,19 +46,20 @@ describe JobPool::Job do
     # pile a bunch of checks into this test so we only have to sleep once
     expect(pool.count).to eq 0
     claimed = nil
+    job = nil
 
     elapsed = time_this_block do
       # echo -n doesn't work here because of platform variations
       # and for some reason jruby requires the explicit subshell; mri launches it automatically
-      process = JobPool::Job.new(pool, '/bin/sh -c "sleep 0.1 && printf done."', chin, chout, cherr)
+      job = JobPool::Job.new(pool, '/bin/sh -c "sleep 0.1 && printf done."', stdin: small_input)
       expect(pool.count).to eq 1
-      process.stop
-      expect(process.start_time).not_to eq nil
-      expect(process.stop_time).not_to eq nil
-      claimed = process.stop_time - process.start_time
-      expect(chout.string).to eq 'done.'
-      expect(process.finished?).to eq true
-      expect(process.success?).to eq true
+      job.stop
+      expect(job.start_time).not_to eq nil
+      expect(job.stop_time).not_to eq nil
+      claimed = job.stop_time - job.start_time
+      expect(job.output).to eq 'done.'
+      expect(job.finished?).to eq true
+      expect(job.success?).to eq true
     end
 
     # ensure process elapsed time is in the ballpark
@@ -69,36 +68,37 @@ describe JobPool::Job do
     expect(claimed).to be <= elapsed
 
     expect(pool.count).to eq 0
-    expect(chout.closed_read?).to eq true
-    expect(cherr.closed_read?).to eq true
+    expect(job.stdout.closed_read?).to eq true
+    expect(job.stderr.closed_read?).to eq true
   end
 
   it "has a working kill method" do
+    job = nil
     elapsed = time_this_block do
-      process = JobPool::Job.new(pool, ['sleep', '0.5'], chin, chout, cherr)
+      job = JobPool::Job.new(pool, ['sleep', '0.5'], stdin: small_input)
 
-      expect(process.finished?).to eq false
-      expect(process.killed?).to eq false
-      expect(process.success?).to eq false
-      expect(process.timed_out?).to eq false
+      expect(job.finished?).to eq false
+      expect(job.killed?).to eq false
+      expect(job.success?).to eq false
+      expect(job.timed_out?).to eq false
 
-      process.kill
+      job.kill
 
-      expect(process.finished?).to eq true
-      expect(process.killed?).to eq true
-      expect(process.success?).to eq false
-      expect(process.timed_out?).to eq false
+      expect(job.finished?).to eq true
+      expect(job.killed?).to eq true
+      expect(job.success?).to eq false
+      expect(job.timed_out?).to eq false
     end
 
     expect(elapsed).to be < 0.5
-    expect(chout.closed_read?).to eq true
-    expect(cherr.closed_read?).to eq true
+    expect(job.stdout.closed_read?).to eq true
+    expect(job.stderr.closed_read?).to eq true
   end
 
   it "handles invalid commands" do
     expect {
       expect(pool.count).to eq 0
-      process = JobPool::Job.new(pool, ['ThisCmdDoes.Not.Exist.'], chin, chout, cherr)
+      job = JobPool::Job.new(pool, ['ThisCmdDoes.Not.Exist.'], stdin: small_input)
       raise "we shouldn't get here"
     }.to raise_error(/[Nn]o such file/)
     expect(pool.count).to eq 0
@@ -106,15 +106,14 @@ describe JobPool::Job do
 
   it "has a working timeout" do
     elapsed = time_this_block do
-      process = JobPool::Job.new(pool, ['sleep', '10'], chin, chout, cherr, 0.1)
+      job = JobPool::Job.new(pool, ['sleep', '10'], stdin: small_input, timeout: 0.1)
     end
     expect(elapsed).to be < 0.2
   end
 
-  # TODO: should probably define exactly what happens in this case
   it "accepts a 0-length timeout" do
     elapsed = time_this_block do
-      process = JobPool::Job.new(pool, ['sleep', '10'], chin, chout, cherr, 0)
+      job = JobPool::Job.new(pool, ['sleep', '10'], stdin: small_input, timeout: 0)
     end
     expect(elapsed).to be < 0.2
   end

data/spec/job_pool_spec.rb

@@ -29,10 +29,10 @@ describe JobPool do
     after { expect(pool.count).to eq 0 }
 
     it "counts and kills multiple processes" do
-      pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
-      pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
-      pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
-      pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
+      pool.launch(['sleep', '20'])
+      pool.launch(['sleep', '20'])
+      pool.launch(['sleep', '20'])
+      pool.launch(['sleep', '20'])
       expect(pool.count).to eq 4
       pool.first.kill
       expect(pool.count).to eq 3
@@ -43,9 +43,9 @@ describe JobPool do
     it "waits for multiple processes" do
       # these sleep durations might be too small, depends on machine load and scheduling.
       # if you're seeing threads finishing in the wrong order, try increasing them 10X.
-      process1 = pool.launch(['sleep', '.3'], StringIO.new, StringIO.new, StringIO.new)
-      process2 = pool.launch(['sleep', '.1'], StringIO.new, StringIO.new, StringIO.new)
-      process3 = pool.launch(['sleep', '.2'], StringIO.new, StringIO.new, StringIO.new)
+      process1 = pool.launch(['sleep', '.3'])
+      process2 = pool.launch(['sleep', '.1'])
+      process3 = pool.launch(['sleep', '.2'])
       expect(pool.count).to eq 3
 
       child = pool.wait_next
@@ -65,7 +65,8 @@ describe JobPool do
     it "handles waiting for zero processes" do
       expect {
         child = pool.wait_next
-      }.to raise_exception(ThreadsWait::ErrNoWaitingThread)
+        # if I don't use a string, rdoc claims ThreadsWait is my class.  Bug?
+      }.to raise_exception(Object.const_get 'ThreadsWait::ErrNoWaitingThread')
     end
   end
 

data/spec/readme_spec.rb

@@ -4,7 +4,7 @@ require 'job_pool'
 describe 'README' do
   it "can do the first example" do
     pool = JobPool.new
-    job = pool.launch("sleep 0.1; tr A-Za-z N-ZA-Mn-za-m", "the secrets")
+    job = pool.launch("sleep 0.1; tr A-Za-z N-ZA-Mn-za-m", stdin: "the secrets")
     expect(job.output).to eq ''
     expect(pool.count).to eq 1
     sleep(0.2)
@@ -16,10 +16,12 @@ describe 'README' do
     pool = JobPool.new
     # can't use `expect { ... }.to output('contents').to_stdout`
     # because the test's stdout gets closed
-    outstr = StringIO.new
-    pool.launch 'gunzip --to-stdout', File.open('spec/contents.txt.gz'), outstr
+    source = File.open('spec/contents.txt.gz')
+    destination = File.open("/tmp/test-#{$$}-out.txt", 'w')
+    pool.launch 'gunzip --to-stdout', stdin: source, stdout: destination
     pool.wait_next
-    expect(outstr.string).to eq "contents\n"
+    expect(File.read "/tmp/test-#{$$}-out.txt").to eq "contents\n"
+    File.delete "/tmp/test-#{$$}-out.txt"
   end
 
   it "can do the killer example" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job_pool
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: '0.6'
 platform: ruby
 authors:
 - Scott Bronson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-10 00:00:00.000000000 Z
+date: 2015-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -104,3 +104,4 @@ test_files:
 - spec/job_pool_spec.rb
 - spec/readme_spec.rb
 - spec/spec_helper.rb
+has_rdoc: