parallel-forkmanager 1.0.1 → 2.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7f3f673b359f8e36f9477332915c1ba239b6a40549e1077340d18139684e69a3
+  data.tar.gz: 33cbc848bd7faa60e297e8d064db80251e1422aae9d13fd3b081bab7169b2ede
+SHA512:
+  metadata.gz: 04a22d48287995e4e7759c5310becad54130d7c1624c2b096ef512dc955cca90a1b75e95fef78298e0269bc008fd31ea34a84a3480922e21059fe030bf057b10
+  data.tar.gz: 948fccecafe9d36a62386a29b01665e92638c956b001b39b9ac142e401f239673a10c52e7bd86e1ab7536b23ad84a629d8f4fc86d3f5a995ebe8286bda11a48b

data/.gitignore

@@ -0,0 +1,6 @@
+/Gemfile.lock
+# packaging artifacts
+/pkg/
+# yard documentation
+/doc/
+/.yardoc/

data/.rubocop.yml

@@ -0,0 +1,17 @@
+---
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+Style/Documentation:
+  # Lots of files under lib/parallel/forkmanager/ start with
+  #
+  # module Parallel
+  #   class ForkManager
+  #     ...
+  #
+  # which wants a comment before the class ForkManager to keep rubocop
+  # happy.
+  #
+  # That should have been documented in lib/parallel/forkmanager.rb so this
+  # exlcude stops those warnings without littering the code with directives.
+  Exclude:
+  - "lib/parallel/forkmanager/**/*.rb"

data/.travis.yml

@@ -0,0 +1,5 @@
+ language: ruby
+ rvm:
+   - 1.9.3-p551
+   - 2.1.5
+   - 2.2.2

data/CHANGELOG.md

@@ -0,0 +1,59 @@
+# Parallel::ForkManager Changelog
+
+## 2.0.5 (2015-05-23)
+
+- Adds reap_finished_children, is_child and is_parent to match Perl PFM 1.14.
+
+## 2.0.3 (2015-05-10)
+
+- Start adding tests
+- Switch to Rubygems packaging
+- Allow use of Rubocop
+
+## 2.0.2 (2015-05-08)
+
+- Fixes bug in garbage collection.
+
+## 2.0.1 (2015-05-07)
+
+- Minor doc fixes.
+- Fixes garbage collection.
+
+## 2.0.0 (2015-05-04)
+
+- Refresh to match changes to Perl PFM 1.12.
+- May the 4th be with you.
+
+## 1.5.1 (2011-03-04)
+
+- Resolves bug #29043 wait_one_child failed to retrieve object.
+- Adds conversion of Object to Hash before serialization to avoid 'singleton can't be dumped' error.
+- Minor documentation changes for initialize().
+
+## 1.5.0 (2011-02-25)
+
+- Implements data structure retrieval as had appeared in Perl Parallel::ForkManager 0.7.6.
+- Removes support for passing Proc to run_on_* methods; now supports blocks instead.
+- Documentation updates and code cleanup.
+
+## 1.2.0 (2010-02-01)
+
+- Resolves bug #27748 finish throws an error when used with start(ident).
+- Adds block support to run_on_start(), run_on_wait(), run_on_finish().
+
+## 1.1.1 (2010-01-05)
+
+- Resolves bug with Errno::ECHILD.
+
+## 1.1.0 (2010-01-01)
+
+- Resolves bug [#27661] forkmanager doesn't fork!.
+- Adds block support to start() w/doc changes for same.
+
+## 1.0.1 (2009-10-24)
+
+- Resolves bug #27328 dies with max procs 1.
+
+## 1.0.0 (2008-11-03)
+
+- Initial release

data/EXAMPLES.yard

@@ -0,0 +1,40 @@
+= Parallel::ForkManager Examples
+
+The programs in the examples directory show the use of the Parallel::ForkManager
+gem.
+
+To run them from this directory use a command like:
+
+    ruby -Ilib examples/data_structures_advanced.rb
+
+The programs are discussed below.
+
+== examples/callbacks.rb
+
+Example of a program using callbacks to get child exit codes.
+
+{include:file:examples/callbacks.rb}
+
+== data_structures_string.rb
+
+In this simple example, each child sends back a string.
+
+{include:file:examples/data_structures_string.rb}
+
+== data_structures_advanced.rb
+
+A second data structure retrieval example demonstrates how children
+decide whether or not to send anything back, what to send and how the
+parent should process whatever is retrieved.
+
+{include:file:examples/data_structures_advanced.rb}
+
+== parallel_http_get.rb
+
+Use multiple workers to fetch data from URLs.
+
+{include:file:examples/parallel_http_get.rb}
+
+== use_pfm.rb
+
+{include:file:examples/use_pfm.rb}

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,136 @@
+# Parallel::ForkManager - A simple parallel processing fork manager.
+
+[![Build Status](https://travis-ci.org/npatwardhan/ruby-parallel-forkmanager.svg?branch#master)](https://travis-ci.org/npatwardhan/ruby-parallel-forkmanager)
+
+## Overview
+
+Parallel::ForkManager is used for operations that you would like to do
+in parallel.  Typical use is a downloader which could be retrieving
+hundreds and/or thousands of files.
+
+Parallel::ForkManager, as its name suggests, uses `fork` to handle parallel
+processing instead of threads.  If you've used `fork` before, you're aware
+that you need to be responsible for managing (i.e. cleaning up) the
+processes that were created as a result of the `fork`.
+
+Parallel::ForkManager handles this for you such that you `start` and
+`finish` without having to worry about child processes along
+the way.  Further, Parallel::ForkManager provides useful callbacks
+that you can use when a child starts and/or finishes, or while you're
+waiting for a child to complete.
+
+The code for a downloader that uses Net::HTTP would look like this:
+
+    require "rubygems"
+    require "net/http"
+    require "forkmanager"
+    
+    my_urls = %w(url1 url2 urlN)
+    
+    max_proc = 30
+    my_timeout = 5
+    
+    pm = Parallel::ForkManager.new(max_proc)
+    
+    my_urls.each do |my_url|
+      pm.start(my_url) && next # blocks until new fork slot is available
+      # doing stuff here with my_url will be in a child
+      url = URI.parse(my_url)
+    
+      begin
+        http = Net::HTTP.new(url.host, url.port)
+        http.open_timeout = http.read_timeout = my_timeout
+        res = http.get(url.path)
+    
+        status = res.code
+        if status.to_i != 200
+          print "Cannot get #{url.path} from #{url.host}!\n"
+          pm.finish(255)
+        else
+          pm.finish(0)
+        end
+    rescue Timeout::Error, Errno::ECONNREFUSED #> e
+      print "*** ERROR: #{my_url}: #{e.message}!\n"
+      pm.finish(255)
+      end
+    end
+    
+    pm.wait_all_children
+
+First you need to instantiate the ForkManager with the `new` constructor.
+You must specify the maximum number of processes to be created. If you
+specify 0, then *no* `fork` will be done; this is good for debugging purposes.
+
+Next, use `pm.start` to do the `fork`. `pm.start` returns `nil` in the child
+process, and child pid in the parent process.  The `&& next` skips the internal
+loop in the parent process.  *Note:* `pm.start` dies if the `fork` fails.
+
+`pm.finish` terminates the child process (assuming a fork was done in the
+`start`).
+
+*Note:* You cannot use `pm.start` if you are already in the child process.
+If you want to manage another set of subprocesses in the child process,
+you must instantiate another Parallel::ForkManager object!
+
+## Bugs and Limitations
+
+Parallel::ForkManager is a Ruby port of Perl Parallel::ForkManager
+1.14.  It was originally ported from Perl Parallel::ForkManager 0.7.5
+but was recently updated to integrate features implemented in Perl
+Parallel::ForkManager versions 0.7.6 - 1.14.  Bug reports and feature
+requests are always welcome.
+
+Do not use Parallel::ForkManager in an environment where other child
+processes can affect the run of the main program, so using this module
+is not recommended in an environment where `fork` / `wait` is already used.
+
+If you want to use more than one copy of the Parallel::ForkManager then
+you have to make sure that all children processes are terminated - before you
+use the second object in the main program.
+
+You are free to use a new copy of Parallel::ForkManager in the child
+processes, although I don't think it makes sense.
+
+## Copyright and License
+
+### Author
+
+Nathan Patwardhan <noopy.org@gmail.com>
+
+### Copyright
+
+Copyright (c) 2008 - 2020 Nathan Patwardhan
+
+### License
+
+Distributes under the same terms as Ruby
+
+## Credits
+
+### Documentation
+
+Nathan Patwardhan <noopy.org@gmail.com>, based on Perl
+Parallel::ForkManager documentation by Noah Robin
+<sitz@onastick.net> and dLux <dlux@dlux.hu>.
+
+### Credits (Perl):
+
+- dLux <dlux@dlux.hu> (author, original Perl module)
+- Gábor Szabó <szabgab@cpan.org>  (co-maintainer)
+- Michael Gang (bug report)
+- Noah Robin <sitz@onastick.net> (documentation tweaks)
+- Chuck Hirstius <chirstius@megapathdsl.net>
+  (callback exit status, original Perl example)
+- Grant Hopwood <hopwoodg@valero.com> (win32 port)
+- Mark Southern <mark_southern@merck.com> (bugfix)
+- Ken Clarke [www.perlprogrammer.net](http://www.perlprogrammer.net)
+  (data structure retrieval)
+
+### Credits (Ruby):
+
+- Robert Klemme <shortcutter@googlemail.com>,
+  David A. Black <dblack@rubypal.com> (general awesomeness)
+- Roger Pack <rogerdpack@gmail.com> (bugfix, fork semantics in start,
+  doc changes)
+- Mike Stok <mike@stok.ca> (test cases, percussion, backing vocals)
+- Akinori MUSHA <email@redacted>

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+require "yard"
+
+task default: "test"
+
+task :test do
+  $LOAD_PATH.unshift "lib", "test"
+  Dir.glob("./test/test_*/**/*.rb") { |f| require f }
+end
+
+OTHER_DOC_FILES = %w(README.md EXAMPLES.yard CHANGELOG.md)
+YARD::Rake::YardocTask.new do |t|
+  # The reason for .md and .yard is that Github won't show the included
+  # files if it's markdown, so this attempts to put the "useful" files in
+  # markdown.
+  t.files   = %w(lib/**/*.rb - README.md CHANGELOG.md EXAMPLES.yard)
+  t.stats_options = ["--list-undoc"]         # optional
+end

data/examples/callbacks.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "parallel/forkmanager"
+
+max_procs = 5
+names = %w(Fred Jim Lily Steve Jessica Bob Dave Christine Rico Sara)
+
+pm = Parallel::ForkManager.new(max_procs)
+
+# Setup a callback for when a child finishes up so we can get its exit code
+pm.run_on_finish do |pid, exit_code, ident|
+  puts "** #{ident} just got out of the pool with PID #{pid} and exit code: #{exit_code}"
+end
+
+pm.run_on_start do |pid, ident|
+  puts "** #{ident} started, pid: #{pid}"
+end
+
+pm.run_on_wait(0.5) do
+  puts "** Have to wait for one children ..."
+end
+
+names.each_index do |child|
+  pm.start(names[child]) && next
+
+  # This code is the child process
+  puts "This is #{names[child]}, Child number #{child}"
+  sleep(2 * child)
+  puts "#{names[child]}, Child #{child} is about to get out..."
+  sleep 1
+  pm.finish(child) # pass an exit code to finish
+end
+
+puts "Waiting for Children..."
+pm.wait_all_children
+puts "Everybody is out of the pool!"

data/examples/data_structures_advanced.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "parallel/forkmanager"
+
+max_procs = 20
+
+pm = Parallel::ForkManager.new(max_procs, "tempdir" => "/tmp")
+
+# data structure retrieval and handling
+retrieved_responses = {} # for collecting responses
+
+# data structure retrieval and handlin
+pm.run_on_finish do |_pid, _exit_code, ident, _exit_signal, _core_dump, data|
+  if data # test rather than assume child sent anything
+    puts "#{ident} returned #{data.inspect}."
+
+    retrieved_responses[ident] = data
+  else
+    puts "#{ident} did not send anything."
+  end
+end
+
+# generate a list of instructions
+instructions = [  # a unique identifier and what the child process should send
+  { "name" => "ENV keys as a string", "send" => "keys" },
+  { "name" => "Send Nothing" },
+  { "name" => "Childs ENV", "send" => "all" },
+  { "name" => "Child chooses randomly", "send" => "random" },
+  { "name" => "Invalid send instructions", "send" => "Na Na Nana Na" },
+  { "name" => "ENV values in an array", "send" => "values" }
+]
+
+# run the parallel processes
+instructions.each do |instruction|
+  # this time we are using an explicit, unique child process identifier
+  pm.start(instruction["name"]) && next
+
+  unless instruction.key?("send")
+    puts "MT name #{instruction['name']}"
+    pm.finish(0)
+  end
+
+  data = case instruction["send"]
+         when "keys"   then ENV.keys
+         when "values" then ENV.values
+         when "all"    then ENV.to_h
+         when "random"
+           ["I'm just a string.",
+            %w(I am an array),
+            { "type" => "associative array",
+              "synonym" => "hash",
+              "cool" => "very :)" }
+           ].sample
+         else
+           "Invalid instructions: #{instruction['send']}"
+         end
+
+  pm.finish(0, data)
+end
+
+pm.wait_all_children
+
+# post fork processing of returned data structures
+retrieved_responses.keys.sort.each do |response|
+  puts "Post processing \"#{response}\"..."
+end

data/examples/data_structures_string.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "parallel/forkmanager"
+
+max_procs = 2
+persons = %w(Fred Wilma Ernie Bert Lucy Ethel Curly Moe Larry)
+
+pm = Parallel::ForkManager.new(max_procs, "tempdir" => "/tmp")
+
+# data structure retrieval and handling
+pm.run_on_finish do |pid, _exit_code, _ident, _exit_signal, _core_dump, data|
+  if data # children are not forced to send anything
+    puts data
+  else  # problems occuring during storage or retrieval will throw a warning
+    puts "No message received from child process #{pid}!"
+  end
+end
+
+# prep random statement components
+foods = [
+  "chocolate", "ice cream", "peanut butter", "pickles", "pizza", "bacon",
+  "pancakes", "spaghetti", "cookies"
+]
+opinions = [
+  "loves", "can't stand", "always wants more", "will walk 100 miles for",
+  "only eats", "would starve rather than eat"
+]
+
+# run the parallel processes
+persons.each do |person|
+  pm.start && next
+
+  # generate a random statement about food preferences
+  statement = "#{person} #{opinions.sample} #{foods.sample}"
+
+  if rand(5) > 0
+    pm.finish(0, statement)
+  else
+    pm.finish(0)
+  end
+end
+
+pm.wait_all_children