team_effort 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: caad20978d09546b723794fedd95ca0d341c1044
-  data.tar.gz: b1fee646c8cae6b0cdf5b10c1441f0df6e454e56
+  metadata.gz: 75c7884021dc7ff32714dd57b1d642ba110817a4
+  data.tar.gz: 0129f103f4db6e91a423dab711c9de64c7094dfb
 SHA512:
-  metadata.gz: ebcae5593ecede37d63f3e6711ba2f8b08a28795d9e85c179107e80993396c236b748a69f9930f2fbe6734c0341149ff13b9154272fe7b799eb946ca7144cde3
-  data.tar.gz: 63f51a5f48c662fceecb4823a6d46f45cea15428b21db0ef2b84d6728a52524e65ea84515a7ef0a36ef8da198b0c676c51442a7c4cdcdb5519adcbfab99505be
+  metadata.gz: 42c57584ea793a16e6888535e29e94ac44aeb4a1a580c001cc4ec48e53b4f4637411bb8584b87c0e7d7e2c838673aa2eb5c733620fc26cd163575d08af4cd642
+  data.tar.gz: 5ff868836356131210df02424b805807cf04bba59a9e717c1340de88248faa724c640b830fef13b05ed91320d83465331e13ad94d127cbdb0de6bc17a1237529

data/README.md

@@ -1,8 +1,23 @@
 # TeamEffort
 
-Team Effort is a module that makes it easy to dispatch work to child
-processes allowing you to speed processing by taking advantage of
-multiple cores.
+TeamEffort makes it easy to process a collection with child processes
+allowing you to take advantage of multiple cores. By replacing
+
+```ruby
+    collection.each do |item|
+      # do some work on item
+    end
+```
+
+with 
+
+```ruby
+    TeamEffort.work(collection) do |item|
+      # do some work on item
+    end
+```
+
+you get each item processed in a new child process.
 
 ## Installation
 
@@ -20,39 +35,72 @@ Or install it yourself as:
 
 ## Usage
 
-To do work in child processes just call `TeamEffort.work` with a collection
-of items to process and a block:
+To do work in child processes just call `TeamEffort.work` with a
+collection of items to process and a block:
 
 ```ruby
-class ProcessALotOfStuff
-
-  def some_method
-    # collection = a lot of stuff from somewhere
     TeamEffort.work(collection) do |item|
       # do some work on item
     end
-  end
-  
-end
 ```
 
 You may specify the number of child processes with the work method:
  
 ```ruby
-def some_method
-  # collection = a lot of stuff from somewhere
-  TeamEffort.work(collection, 3) do |item| # do the work using 3 child processes
-    # do some work on item
-  end
-end
+    TeamEffort.work(collection, 3) do |item| # do the work using 3 child processes
+      # do some work on item
+    end
 ```
 
 The number of child processes defaults to 4.
 
-The work method will create a new child process for each item in the
-enumeration using ruby's Process.fork so there is overhead on each
-item processed. Team Effort works best when there is substantial work
-to be performed on each item to minimize overhead.
+In rails you need to reestablish your ActiveRecord connection in the
+child process:
+
+```ruby
+    ActiveRecord::Base.clear_all_connections!
+    begin
+      TeamEffort.work(collection, max_process_count) do |item|
+    
+        ActiveRecord::Base.establish_connection
+    
+        # do some work with active record
+    
+      end
+    ensure
+      ActiveRecord::Base.establish_connection
+    end    
+```
+
+Logging can be messy when multiple processes are writing to the same
+io channel. One approach is to wrap logging statements in a
+synchronize block:
+
+```ruby
+    def pid_logger(msg)
+      @mutex ||= Mutex.new
+      @mutex.synchronize do
+        puts "[#{Process.pid} at #{Time.now.strftime('%H:%M:%S')}] #{msg}"
+      end
+    end
+```
+
+## Discussion
+
+TeamEffort uses child processes to do concurrent processing. To review
+the unix process model I recommend Jesse Storimer's
+[Working With Unix Processes][1].
+ 
+[1]: http://www.jstorimer.com/products/working-with-unix-processes
+
+A disadvantage to using child processes for concurrent processing is
+the work required to create the child process and the duplication of
+memory.  For this reason you should only use TeamEffort on collections
+where there is significant processing to be performed on each item.
+
+An advantage of using child processes is that the memory is reclaimed
+when the child process goes away. This can make long running jobs
+resilient to memory leaks.
 
 ## Contributing
 

data/lib/team_effort.rb

@@ -1,11 +1,11 @@
-require "team_effort/version"
+require_relative "team_effort/version"
 
 module TeamEffort
   def self.work(enumerable, max_process_count = 4)
     pids = []
 
     enumerable.each do |args|
-      if pids.size == max_process_count
+      while pids.size == max_process_count
         finished_pid = Process.wait
         pids.delete finished_pid
       end
@@ -20,4 +20,4 @@ module TeamEffort
       pids.delete finished_pid
     end
   end
-end
+end

data/lib/team_effort/version.rb

@@ -1,3 +1,3 @@
 module TeamEffort
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/test/test_team_effort.rb

@@ -29,5 +29,71 @@ describe TeamEffort do
         line.must_match /^\d+$/
       end
     end
+
+
+    it 'ignores other child process completions' do
+      output_io_class = Class.new do
+        def initialize
+          @mutex = Mutex.new
+          @io = Tempfile.new('mumble')
+        end
+
+        def log(text)
+          @mutex.synchronize do
+            @io.puts text
+            @io.flush
+          end
+        end
+
+        def lines
+          @io.rewind
+          @io.read.split(/\n/)
+        end
+      end
+
+      output_io = output_io_class.new
+
+      require 'socket'
+
+      unmanaged_child_reader, unmanaged_child_writer = Socket.pair(:UNIX, :DGRAM, 0)
+      maxlen = 1000
+
+      fork do
+        output_io.log "unmanaged starting"
+        unmanaged_child_writer.close
+        output_io.log "unmanaged waiting for IO"
+        message = unmanaged_child_reader.recv(maxlen)
+        output_io.log "unmanaged received >#{message}< and exiting"
+        output_io.log "unmanaged finishing"
+      end
+
+      sleep 1
+
+      TeamEffort.work([1, 2], 1) do |index|
+        output_io.log "task #{index} starting"
+        unmanaged_child_reader.close
+        if index == 1
+          output_io.log "task 1 waking unmanaged process"
+          unmanaged_child_writer.send("wake up", 0)
+          unmanaged_child_writer.close
+          sleep 1
+        end
+        output_io.log "task #{index} finishing"
+      end
+
+      lines = output_io.lines
+
+      lines.must_equal [
+                         'unmanaged starting',
+                         'unmanaged waiting for IO',
+                         'task 1 starting',
+                         'task 1 waking unmanaged process',
+                         'unmanaged received >wake up< and exiting',
+                         'unmanaged finishing',
+                         'task 1 finishing',
+                         'task 2 starting',
+                         'task 2 finishing',
+                       ]
+    end
   end
 end

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: team_effort
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Kelly Felkins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-12 00:00:00.000000000 Z
+date: 2015-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: "    Team Effort provides a simple wrapper to ruby's process management
@@ -46,9 +46,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-gemset
-- .ruby-version
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
 - LICENSE
 - README.md
 - Rakefile
@@ -66,17 +66,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Use child processes to process a collection in parallel