queue_map 0.1

data/MIT_LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 LeadTune
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/queue_map_consumer

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+options = {}
+opt_p = OptionParser.new do |opts|
+  opts.banner = "Usage: script/export_sales.rb [options] path/to/consumer.rb"
+  opts.separator "Options:"
+  opts.on('-f', '--foreground', "Run appraiser in the foreground") do
+    options[:foreground] = true
+  end
+  opts.on_tail("-h", "--help", "Show this message")
+end
+opt_p.parse!
+consumer_path = ARGV[0]
+if consumer_path.nil?
+  puts opt_p
+  exit 1
+end
+
+require "rubygems"
+require File.dirname(__FILE__) + "/../lib/queue_map"
+require File.dirname(__FILE__) + "/../lib/queue_map/consumer"
+
+consumer = QueueMap::Consumer.from_file(consumer_path, :strategy => :fork)
+
+unless options[:foreground]
+  # daemonize
+  puts "Daemonizing"
+  pid = Kernel.fork
+  if pid
+    Process.detach(pid)
+    puts "Daemonized as #{pid}"
+    exit 0
+  else
+    Signal.trap('HUP', 'IGNORE') # Don't die upon logout
+  end
+end
+puts "Starting consumers..."
+consumer.start

data/lib/ext/enumerable.rb

@@ -0,0 +1,5 @@
+module Enumerable
+  def queue_map(*args)
+    QueueMap.map(self, *args)
+  end
+end

data/lib/queue_map/consumer.rb

@@ -0,0 +1,153 @@
+class QueueMap::Consumer
+  attr_accessor :count_workers, :worker_proc, :on_exception_proc
+  attr_reader :name
+  class Configurator
+    def initialize(base)
+      @base = base
+    end
+
+    def before_fork(&block);        @base.before_fork_procs       << block; end
+    def after_fork(&block);         @base.after_fork_procs        << block; end
+    def between_responses(&block);  @base.between_responses_procs << block; end
+    def worker(&block);             @base.worker_proc             =  block; end
+    def on_exception(&block);       @base.on_exception_proc       =  block; end
+    def count_workers(value);       @base.count_workers           =  value; end
+  end
+
+
+  def self.new_from_block(name, options = { }, &block)
+    consumer = new(name, options)
+    Configurator.new(consumer).instance_eval(&block)
+    consumer
+  end
+
+  def self.from_file(consumer_path, options = { })
+    name = File.basename(consumer_path).gsub(/_consumer\.rb$/, '').to_sym
+    consumer = new(name, options)
+    Configurator.new(consumer).instance_eval(File.read(consumer_path), consumer_path, 1)
+    consumer
+  end
+
+  def initialize(name, options = { })
+    @name = name
+    case options[:strategy]
+    when :fork
+      extend(ForkStrategy)
+    when :thread
+      extend(ThreadStrategy)
+    when :test
+      nil
+    else
+      raise "Invalid strategy: #{options[:strategy]}"
+    end
+  end
+
+  def after_fork_procs
+    @after_fork_procs ||= []
+  end
+
+  def before_fork_procs
+    @before_fork_procs ||= []
+  end
+
+  def between_responses_procs
+    @between_responses_procs ||= []
+  end
+
+  def start
+    raise RuntimeError, "Called start on Consumer without strategy"
+  end
+
+  def stop
+    raise RuntimeError, "Called stop on Consumer without strategy"
+  end
+
+  def run_consumer
+    QueueMap.with_bunny do |bunny|
+      q = bunny.queue(name.to_s, :durable => false, :auto_delete => true)
+      while msg = q.pop
+        # STDOUT << "[#{Thread.current[:id]}]"
+        return if @shutting_down
+        begin
+          (sleep 0.05; next) if msg == :queue_empty
+          msg = Marshal.load(msg)
+          result = worker_proc.call(msg[:input])
+          bunny.queue(msg[:response_queue]).publish(Marshal.dump(:result => result, :index => msg[:index]))
+          between_responses_procs.each { |p| p.call }
+        rescue Exception => e
+          if on_exception_proc
+            on_exception_proc.call(e)
+          else
+            puts e
+            puts e.backtrace
+          end
+        end
+      end
+    end
+  end
+
+  module ThreadStrategy
+    def start
+      @threads = []
+      count_workers.times do |c|
+        @threads << (Thread.new do
+                       begin
+                         run_consumer
+                       rescue Exception => e
+                         puts %(#{e}\n#{e.backtrace.join("\n")})
+                       end
+                     end)
+      end
+    end
+
+    def stop(graceful = true)
+      if graceful
+        @shutting_down = true
+      else
+        @threads.each { |t| t.kill }
+      end
+    end
+  end
+
+  module ForkStrategy
+    class ImaChild < Exception; end
+
+    def start
+      @child_pids = []
+      @master_pid = Process.pid
+      before_fork_procs.each { |p| p.call }
+      begin
+        (count_workers - 1).times do |c|
+          pid = Kernel.fork
+          raise ImaChild if pid.nil?
+          @child_pids << pid
+        end
+        $0 = "#{name} queue_map consumer: master"
+      rescue ImaChild
+        $0 = "#{name} queue_map consumer: child"
+        # Parent Child Testing Product was a success! http://is.gd/fXmmZ
+      end
+      Signal.trap("TERM") { stop(false) }
+      Signal.trap("INT") { stop }
+      after_fork_procs.each { |p| p.call }
+      run_consumer
+    end
+
+    def stop(graceful = true)
+      @child_pids.each do |pid|
+        begin
+          Process.kill(graceful ? "INT" : "TERM", pid)
+        rescue Errno::ESRCH => e
+          puts "Unable to signal process #{pid}. Does the process not exist?"
+        end
+      end
+
+      puts "#{Process.pid}: stopping (graceful: #{graceful})"
+      if graceful
+        @shutting_down = true
+      else
+        exit 0
+      end
+    end
+  end
+end

data/lib/queue_map/version.rb

@@ -0,0 +1,3 @@
+module QueueMap
+  VERSION=0.1
+end

data/lib/queue_map.rb

@@ -0,0 +1,98 @@
+require File.dirname(__FILE__) + "/ext/enumerable.rb"
+require "bunny"
+require 'timeout'
+
+module QueueMap
+  autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
+  BUNNY_MUTEX = Mutex.new
+  extend self
+  attr_accessor :mode
+  attr_writer :consumer_base_path
+  attr_accessor :consumer_path
+
+  DEFAULT_ON_TIMEOUT = lambda { |r| nil }
+
+  def unique_name
+    @unique_name ||= "#{`hostname`.chomp}-#{Process.pid}-#{Time.now.usec}"
+  end
+
+  def consumer_base_path
+    @consumer_base_path ||= "lib/consumers"
+  end
+
+  def consumer_path
+    @consumer_path ||= Hash.new do |hash, key|
+      File.join(consumer_base_path, "#{key}_consumer.rb")
+    end
+  end
+
+  def response_queue_name(name)
+    @inc ||= 0
+    "#{name}_response_#{unique_name}_#{@inc += 1}"
+  end
+
+  def map(collection, name, options = {})
+    return queue_map_internal(collection, name) if mode == :test
+
+    with_bunny do |bunny|
+      q = bunny.queue(name.to_s)
+      response_queue_name = response_queue_name(name)
+      response_queue = bunny.queue(response_queue_name, :durable => false, :exclusive => true, :auto_delete => true)
+
+      (0..(collection.length - 1)).each do |i|
+        q.publish(Marshal.dump(:input => collection[i], :index => i, :response_queue => response_queue_name))
+      end
+
+      results = {}
+      begin
+        Timeout.timeout(options[:timeout] || 5) do
+          collection.length.times do
+            sleep 0.05 while (next_response = response_queue.pop) == :queue_empty
+            response = Marshal.load(next_response)
+            results[response[:index]] = response[:result]
+          end
+        end
+      rescue Timeout::Error => e
+      end
+
+      (0..(collection.length - 1)).map do |i|
+        results[i] || (options[:on_timeout] || DEFAULT_ON_TIMEOUT).call(collection[i])
+      end
+    end
+  end
+
+  def queue_map_internal(collection, name, *args)
+    collection.map(&consumer(name).worker_proc)
+  end
+
+  def consumers
+    @consumers ||= { }
+  end
+
+  def consumer(name)
+    consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :thread)
+  end
+
+  def with_bunny(&block)
+    bunny = nil
+    BUNNY_MUTEX.synchronize do
+      bunny = Bunny.new((@connection_info || { }).merge(:spec => '08'))
+      bunny.start
+    end
+    begin
+      yield bunny
+    ensure
+      bunny.close_connection unless bunny.status == :not_connected
+    end
+  end
+
+  def connection_info=(connection_info)
+    @bunny = nil
+    @connection_info = connection_info
+  end
+
+  def mode=(mode)
+    @mode = mode
+    consumers.clear
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification 
+name: queue_map
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Tim Harper
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-11 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 0
+        - 5
+        - 3
+        version: 0.5.3
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 7712058
+        segments: 
+        - 2
+        - 0
+        - 0
+        - rc
+        version: 2.0.0.rc
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: background_process
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: ruby-debug
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Queue Map
+email: 
+- tim@leadtune.com
+executables: 
+- queue_map_consumer
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/queue_map_consumer
+- lib/ext/enumerable.rb
+- lib/queue_map/consumer.rb
+- lib/queue_map/version.rb
+- lib/queue_map.rb
+- MIT_LICENSE
+has_rdoc: true
+homepage: http://github.com/leadtune/queue_map
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Map a task across a pool of consumers, and recieve the result when the consumers are done
+test_files: []
+