parallel_work 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in parallel_work.gemspec
+gemspec

data/README.md

@@ -0,0 +1,22 @@
+parallel_work
+===========
+
+Spread work over multiple processes on the same server, with a central work queue.
+
+## Installation
+
+    gem install parallel_work
+
+## Example Usage
+
+```ruby
+require "parallel_work"
+
+work = (1..400).to_a
+
+ParallelWork.process work, 8 do |data|
+  puts data
+  sleep rand
+end
+
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/example/example.rb

@@ -0,0 +1,11 @@
+require File.expand_path "../lib/parallel_work", File.dirname(__FILE__)
+
+# Turn on REE copy-on-write GC if we're running on REE
+GC.copy_on_write_friendly = true if GC.respond_to? :copy_on_write_friendly?
+
+work = (1..400).to_a
+
+ParallelWork.process work, 8 do |data|
+  puts data
+  sleep rand
+end

data/lib/parallel_work.rb

@@ -0,0 +1,15 @@
+require File.expand_path "./parallel_work/version", File.dirname(__FILE__)
+require 'socket'
+require File.expand_path './parallel_work/runner', File.dirname(__FILE__)
+require File.expand_path './parallel_work/messaging', File.dirname(__FILE__)
+require File.expand_path './parallel_work/message', File.dirname(__FILE__)
+
+module ParallelWork
+
+  # @param work [#each]
+  # @params worker [Fixnum] number of worker processes
+  def self.process work, workers, &process_block
+    Runner.new(work.each, process_block).spawn(workers).close_other_sockets.start
+  end
+
+end

data/lib/parallel_work/message.rb

@@ -0,0 +1,30 @@
+Dir[File.dirname(__FILE__) + '/message/*.rb'].each do |file|
+  require file
+end
+
+module ParallelWork
+  class Message
+
+    def self.build name
+      class_name = name[0,1].upcase + name[1,name.length].downcase
+      const_get(class_name).new
+    end
+
+    def name
+      raise "To be implemented in subclass"
+    end
+
+    def has_payload?
+      raise "To be implemented in subclass"
+    end
+
+    def payload
+      nil
+    end
+
+    def payload_length
+      payload ? payload.length : 0
+    end
+
+  end
+end

data/lib/parallel_work/message/quit.rb

@@ -0,0 +1,15 @@
+module ParallelWork
+  class Message
+    class Quit < Message
+
+      def name
+        "QUIT"
+      end
+
+      def has_payload?
+        false
+      end
+
+    end
+  end
+end

data/lib/parallel_work/message/ready.rb

@@ -0,0 +1,15 @@
+module ParallelWork
+  class Message
+    class Ready < Message
+
+      def name
+        "READY"
+      end
+
+      def has_payload?
+        false
+      end
+
+    end
+  end
+end

data/lib/parallel_work/message/work.rb

@@ -0,0 +1,27 @@
+module ParallelWork
+  class Message
+    class Work < Message
+
+      def initialize data=nil
+        @data = data
+      end
+
+      def name
+        "WORK"
+      end
+
+      def has_payload?
+        true
+      end
+
+      def payload
+        @data
+      end
+
+      def payload= data
+        @data = data
+      end
+
+    end
+  end
+end

data/lib/parallel_work/messaging.rb

@@ -0,0 +1,113 @@
+module ParallelWork
+  class Messaging
+
+    MESSAGE_TYPE_LENGTH = 5
+
+    # @param [Socket]
+    # @param [ParallelWork::Message]
+    # @return [void]
+    def self.send socket, message
+      Sending.send socket, message
+    end
+
+    # @param [Socket]
+    # @return [ParallelWork::Message]
+    def self.recv socket
+      Receiving.recv socket
+    end
+
+    private
+
+    class Sending
+
+      def self.send socket, message
+        new(socket, message).send_message
+      end
+
+      def initialize socket, message
+        @socket = socket
+        @message = message
+      end
+
+      def send_message
+        send_message_type
+        if @message.has_payload?
+          send_payload_length
+          send_payload
+        end
+      end
+
+      private
+
+      def send_message_type
+        send "%#{MESSAGE_TYPE_LENGTH}s" % @message.name
+      end
+
+      # payload length sent as 32 bit int.
+      def send_payload_length
+        send [marshalled_payload.length].pack('l')
+      end
+
+      def send_payload
+        send marshalled_payload
+      end
+
+      def send data
+        @socket.send(data, 0)
+      end
+
+      def marshalled_payload
+        @marshalled_payload ||= Marshal.dump(@message.payload)
+      end
+
+    end
+
+    class Receiving
+
+      def self.recv socket
+        new(socket).recv_message
+      end
+
+      def initialize socket
+        @socket = socket
+      end
+
+      def recv_message
+        message_type = recv_message_type.strip
+        message = build_message message_type
+        if message.has_payload?
+          payload_length = recv_payload_length
+          marshalled_payload = recv_payload payload_length
+          unmarshalled_payload = unmarshall_payload marshalled_payload
+          message.payload = unmarshalled_payload
+        end
+        message
+      end
+
+      private
+
+      def recv_message_type
+        @socket.recv(MESSAGE_TYPE_LENGTH)
+      end
+
+      def build_message message_type
+        Message.build(message_type)
+      end
+
+      def recv_payload_length
+        data = @socket.recv(4)
+        data.unpack('l')[0]
+      end
+
+      def recv_payload payload_length
+        @socket.recv(payload_length)
+      end
+
+      def unmarshall_payload marshalled_payload
+        Marshal.load(marshalled_payload)
+      end
+
+    end
+
+  end
+end

data/lib/parallel_work/runner.rb

@@ -0,0 +1,90 @@
+module ParallelWork
+  class Runner
+
+    SOCKET_MAX_LEN = 1000
+    MESSAGE_LEN = 5
+
+    # @param work [#next]
+    def initialize work, process_block
+      @work = work
+      @process_block = process_block
+      @master = nil
+      @parent_sockets = []
+    end
+
+    def setup_sockets
+      @child_socket, parent_socket = Socket.pair("AF_UNIX", "SOCK_DGRAM", 0)
+      @parent_sockets << parent_socket
+    end
+
+    def spawn n_workers
+      setup_sockets
+
+      child_pid = fork
+      if child_pid
+        if n_workers == 1
+          @master = true
+        else
+          @master = false
+          spawn n_workers - 1
+        end
+      else
+        @master = false
+      end
+
+      self
+    end
+
+    def close_other_sockets
+      if @master
+        @child_socket.close
+      else
+        @parent_sockets.each do |socket|
+          socket.close
+        end
+      end
+      self
+    end
+
+    def start
+      if @master
+        puts "master pid #{Process.pid}"
+        its_over = false
+        while !@parent_sockets.empty? && (ready = IO.select(@parent_sockets))
+          socket = ready[0][0]
+          message_from_worker = Messaging.recv socket
+          send_quit = lambda do |socket|
+            Messaging.send socket, Message::Quit.new
+            socket.close
+          end
+          begin
+            if its_over
+              send_quit[socket]
+            else
+              Messaging.send socket, Message::Work.new(@work.next)
+            end
+          rescue StopIteration
+            its_over = true
+            send_quit[socket]
+          end
+          @parent_sockets = @parent_sockets.reject{|s|s.closed?}
+        end
+      else
+        puts "child pid #{Process.pid}"
+        Messaging.send @child_socket, Message::Ready.new
+        while (message = Messaging.recv @child_socket)
+          case message
+            when Message::Work
+              @process_block.call(message.payload)
+              Messaging.send(@child_socket, Message::Ready.new)
+            when Message::Quit
+              exit 0
+            else
+              raise "unknown message #{message}"
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/parallel_work/version.rb

@@ -0,0 +1,3 @@
+module ParallelWork
+  VERSION = "0.0.1"
+end

data/parallel_work.gemspec

@@ -0,0 +1,16 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "parallel_work/version"
+
+Gem::Specification.new do |s|
+  s.name        = "parallel_work"
+  s.version     = ParallelWork::VERSION
+  s.authors     = ["Joel Plane"]
+  s.email       = ["joel.plane@gmail.com"]
+  s.homepage    = "https://github.com/joelplane/parallel_work"
+  s.summary     = %q{Spread work over multiple processes}
+  s.description = %q{When work is in a simple queue, this library allows easily spreading the work over multiple processes on the same server communicating with UNIX sockets. Only tested on Ruby 1.8.7 (REE)}
+
+  s.files         = `git ls-files`.split("\n")
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification 
+name: parallel_work
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Joel Plane
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-01-30 00:00:00 Z
+dependencies: []
+
+description: When work is in a simple queue, this library allows easily spreading the work over multiple processes on the same server communicating with UNIX sockets. Only tested on Ruby 1.8.7 (REE)
+email: 
+- joel.plane@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- example/example.rb
+- lib/parallel_work.rb
+- lib/parallel_work/message.rb
+- lib/parallel_work/message/quit.rb
+- lib/parallel_work/message/ready.rb
+- lib/parallel_work/message/work.rb
+- lib/parallel_work/messaging.rb
+- lib/parallel_work/runner.rb
+- lib/parallel_work/version.rb
+- parallel_work.gemspec
+homepage: https://github.com/joelplane/parallel_work
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Spread work over multiple processes
+test_files: []
+