queue_dispatcher 1.1.0

data/lib/queue_dispatcher/serialization/active_record.rb

@@ -0,0 +1,20 @@
+if defined?(ActiveRecord)
+  class ActiveRecord::Base
+    yaml_as "tag:ruby.yaml.org,2002:ActiveRecord"
+
+    def self.yaml_new(klass, tag, val)
+      if ActiveRecord::VERSION::MAJOR == 3
+        klass.unscoped.find(val['attributes'][klass.primary_key])
+      else # Rails 2
+        klass.with_exclusive_scope { klass.find(val['attributes'][klass.primary_key]) }
+      end
+    rescue ActiveRecord::RecordNotFound
+      #raise QueueDispatcher::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass} , primary key: #{val['attributes'][klass.primary_key]} "
+      '----- QueueDispatcher::DeserializationError -----'
+    end
+
+    def to_yaml_properties
+      ['@attributes']
+    end
+  end
+end

data/lib/queue_dispatcher/syck_ext.rb

@@ -0,0 +1,34 @@
+class Module
+  yaml_as "tag:ruby.yaml.org,2002:module"
+
+  def self.yaml_new(klass, tag, val)
+    val.constantize
+  end
+
+  def to_yaml(options = {})
+    YAML.quick_emit(nil, options) do |out|
+      out.scalar(taguri, name, :plain)
+    end
+  end
+
+  def yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    name
+  end
+end
+
+class Class
+  yaml_as "tag:ruby.yaml.org,2002:class"
+  remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
+end
+
+class Struct
+  def self.yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    "Struct::#{ name }"
+  end
+end

data/lib/queue_dispatcher/version.rb

@@ -0,0 +1,3 @@
+module QueueDispatcher
+  VERSION = "1.1.0"
+end

data/lib/queue_dispatcher/yaml_ext.rb

@@ -0,0 +1,10 @@
+# These extensions allow properly serializing and autoloading of
+# Classes, Modules and Structs
+
+require 'yaml'
+if YAML.parser.class.name =~ /syck|yecht/i
+  require File.expand_path('../syck_ext', __FILE__)
+  require File.expand_path('../serialization/active_record', __FILE__)
+else
+  require File.expand_path('../psych_ext', __FILE__)
+end

data/lib/queue_dispatcher.rb

@@ -0,0 +1,20 @@
+require 'rails'
+require 'queue_dispatcher'
+require 'queue_dispatcher/acts_as_task'
+require 'queue_dispatcher/acts_as_task_queue'
+require 'queue_dispatcher/acts_as_task_controller'
+require 'queue_dispatcher/deserialization_error'
+require 'queue_dispatcher/message_sending'
+require 'queue_dispatcher/qd_logger'
+require 'queue_dispatcher/rc_and_msg'
+require 'queue_dispatcher/yaml_ext'
+
+module QueueDispatcher
+  class Engine < ::Rails::Engine
+  end
+end
+
+ActiveRecord::Base.send(:include, QueueDispatcher::ActsAsTask)
+ActiveRecord::Base.send(:include, QueueDispatcher::ActsAsTaskQueue)
+ActionController::Base.send(:include, QueueDispatcher::ActsAsTaskController)
+Object.send(:include, QueueDispatcher::MessageSending)

data/lib/tasks/queue_dispatcher.rake

@@ -0,0 +1,7 @@
+namespace :queue_dispatcher do
+  desc "Sync extra files from QueueDispatcher gem."
+  task :sync do
+    system "rsync -ruv #{File.dirname(__FILE__)}/../../script #{Rails.root}"
+    system "rsync -ruv #{File.dirname(__FILE__)}/../../rgloader #{Rails.root}"
+  end
+end

data/queue_dispatcher.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "queue_dispatcher/version"
+
+Gem::Specification.new do |s|
+  s.name        = "queue_dispatcher"
+  s.version     = QueueDispatcher::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Philip Kurmann"]
+  s.email       = ["philip.kurmann@inwork.ch"]
+  s.homepage    = ""
+  s.summary     = %q{This Gem installs a queue dispatcher for handling asynchronous tasks}
+  s.description = %q{The QueueDispatcher let you execute queues as workers.}
+
+  s.add_dependency "sys-proctable", '>= 0.9.1'
+  s.add_dependency "deadlock_retry"
+  s.add_dependency "spawn", '>= 1.0.0'
+  s.add_dependency "haml"
+  s.add_dependency "will_paginate"
+  s.add_dependency "jquery-rails"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/script/queue_worker_dispatcher

@@ -0,0 +1,259 @@
+#!/usr/bin/env ruby
+# BEGIN - rails runner… with a relative path
+DAEMON_ARGV = ARGV.clone
+ARGV.clear
+ARGV << 'runner'
+ARGV << '/dev/null'
+$: << File.dirname(__FILE__)
+load 'rails'
+# END - rails runner… with a relative path
+
+require 'sys/proctable'
+include Sys
+include Spawn
+include IwLogger
+
+
+# Stop Active Record Logging
+ActiveRecord::Base.logger = Logger.new(nil)
+ActiveRecord::Base.clear_active_connections!
+
+# ---------------------------------------------------------------------
+# CONFIGURATION
+# ---------------------------------------------------------------------
+$daemon = {
+  :name => "--- - Worker Dispatcher",                                                  # daemon name
+  :process_prefix => "---",                                                            # Prefix for process name
+  :copyright => "(c) 2011 In&Work AG",                                                 # daemon copyright
+  :author => "Philip Kurmann",                                                         # dameon author
+  :version => "1.0",                                                                   # actual version
+  :log_file => "#{File.expand_path(Rails.root)}/log/queue_dispatcher.log",             # log path
+  :pid_file => "#{File.expand_path(Rails.root)}/tmp/pids/queue_worker_dispatcher.pid", # process id path
+  :task_queue_model => TaskQueue,                                                      # TaskQueue
+  :monitoring_sleep => 30,                                                             # seconds
+  :worker_sleep => 15,                                                                 # seconds
+  :worker_count => AppConfig.max_task_queue_workers,                                   # Worker_count
+  :background => false,                                                                # background mode
+  :work => true,                                                                       # daemon work flag
+  :logger_msg_prefix => nil,                                                           # Prefix for logging
+  :worker_pids => []                                                                   # Remember PIDs
+}
+
+$worker = {
+  :name => "--- - Queue Worker",                                                       # Worker name
+  :process_prefix => "---",                                                            # Prefix for process name
+  :work => true,                                                                       # Worker work flag
+  :task_queue => nil                                                                   # Remember TaskQueue of worker, nil means idle
+}
+
+
+# Write a log message
+def daemon_log args = {}
+  msg = args[:msg]
+  msg = "#{$daemon[:logger_msg_prefix]}: #{msg}" unless $daemon[:logger_msg_prefix].blank?
+  sev = args[:sev] || :info
+  log :msg => msg, :sev => sev, :print_log => (! $daemon[:background] || sev == :error)
+end
+
+
+# Set process name
+def daemon_set_process_name name
+  $0 = name
+end
+
+
+# Write PID-File
+def daemon_write_pid_file
+  begin
+    File.open($daemon[:pid_file], "w") { |f| f.write(Process.pid) }
+  rescue Errno::EACCES
+    daemon_log :msg => "Cannot create PID file!", :sev => :error
+    exit
+  end
+end
+
+
+# Clean up before a daemon terminates
+def daemon_clean_up
+  File.delete($daemon[:pid_file])
+end
+
+
+# Install signal handler
+def install_signal_handler signals, args = {}
+  sigtrap = proc do
+    daemon_log :msg => args[:trap_msg] if args[:trap_msg]
+    yield
+  end
+  signals.each { |signal| trap signal, sigtrap }
+  args[:ignore_signals].each { |signal| trap signal, 'IGNORE' } if args[:ignore_signals]
+end
+
+
+# Install signal handler for daemon
+def daemon_install_signal_handler
+  install_signal_handler(['TERM', 'INT', 'HUP'], :trap_msg => "Caught trap signal. Shutdown workers...") do
+    $daemon[:worker_pids].each do |pid|
+      begin
+        Process.kill('TERM', pid)
+      rescue => exception
+      end
+    end
+    $daemon[:work] = false
+  end
+end
+
+
+# Install signal handler for worker
+def worker_install_signal_handler
+  install_signal_handler(['TERM'], :trap_msg => "Caught trap signal. Shutdown...", :ignore_signals => ['HUP', 'INT']) do
+    # Shutdown task_queue
+    $worker[:task_queue].update_attributes :state => 'shutdown' if $worker[:task_queue]
+    $worker[:work] = false
+  end
+end
+
+
+def daemon_runner
+  Lock.release_all
+  daemon_set_process_name "#{$daemon[:process_prefix]}_WorkerDispatcher"
+  daemon_write_pid_file
+  daemon_install_signal_handler
+  spawn_and_monitor_workers
+  # Wait for all children (workers) to be finished
+  Process.waitall
+  daemon_clean_up
+end
+
+
+# Start running
+def daemon_start
+  @logger = Logger.new($daemon[:log_file], 'weekly')
+  $daemon[:logger_msg_prefix] = 'TaskQueueDispatcher'
+
+  if File.exist?($daemon[:pid_file]) then
+    old_pid = IO.read($daemon[:pid_file]).to_i rescue nil
+    ps = Sys::ProcTable.ps(old_pid)
+
+    # Asume, that if the command of the 'ps'-output is 'ruby', the process is still running
+    if ps && (ps.comm == 'ruby')
+      daemon_log :msg => "Process already running!", :sev => :error
+      exit
+    else
+      File.delete($daemon[:pid_file])
+    end
+  end
+ 
+  daemon_log :msg => "Starting process..."
+  if $daemon[:background]
+    spawn_block { daemon_runner }
+  else
+    daemon_runner
+  end
+end
+
+
+# Start an amount of workers
+def spawn_and_monitor_workers
+  daemon_log :msg => "Spawning #{$daemon[:worker_count]} workers..."
+
+  while $daemon[:work]
+    # (Re)start workers
+    while $daemon[:worker_pids].count < $daemon[:worker_count] do
+      sp = spawn_block(:argv => $worker[:process_prefix]) do
+        worker_runner
+      end
+      $daemon[:worker_pids] << sp.handle
+    end
+
+    sleep $daemon[:monitoring_sleep]
+
+    # Delete PIDs from the array child_pids which don't exists anymore
+    $daemon[:worker_pids].each do |ch_pid|
+      begin
+        ps = ProcTable.ps(ch_pid)
+      rescue
+        ps = nil
+        daemon_log :msg => "Error in ProcTable.ps: #{$!}", :sev => :error
+      end
+      $daemon[:worker_pids].delete ch_pid unless ps && ps.comm == 'ruby'
+    end
+  end
+end
+
+
+# Start the task_list dispatcher, which is permanentely looking for task_lists to start
+def worker_runner
+  $daemon[:logger_msg_prefix] = 'TaskQueueWorker'
+
+  worker_install_signal_handler
+  daemon_log :msg => "Start working..."
+
+  # Fetch pending task queues and start working
+  while $worker[:work] do
+    $worker[:task_queue] = $daemon[:task_queue_model].get_next_pending
+
+    if $worker[:task_queue]
+      # Set Process Name and execute tasks
+      $0 = "#{$worker[:process_prefix]}_#{$worker[:task_queue].id}"
+      $worker[:task_queue].run!(:print_log => ! $daemon[:background])
+    else
+      # Set Process Name and sleep
+      $0 = "#{$worker[:process_prefix]}_idle"
+      sleep 15
+    end
+  end
+
+  daemon_log :msg => "Ended!"
+rescue => exception
+  daemon_log :msg => "Fatal error in method 'worker_runner': #{$!}\n#{exception.backtrace}", :sev => :error
+end
+
+
+# Show version
+def daemon_show_version
+  puts "#{$daemon[:name]} v#{$daemon[:version]}"
+  puts $daemon[:copyright]
+end
+
+
+# Show help on command line
+def daemon_show_usage
+  daemon_show_version
+  puts "\nUsage:"
+  puts "    -b, --background        work in background mode"
+  puts "    -v, --version           view version of daemon"
+  puts "    -h, --help              view this help"
+end
+
+
+# Parse command line options 
+def daemon_parse_opts
+  start = true
+
+  unless DAEMON_ARGV.length == 0 
+    case DAEMON_ARGV[0]
+      when '-b', '--background'
+        $daemon[:background] = true;
+   
+      when '-v', '--version'
+        daemon_show_version
+        start = false
+   
+      when '-h', '--help'
+        daemon_show_usage
+        start = false
+   
+      else
+        puts "Invalid argument: #{DAEMON_ARGV[0]}" if !DAEMON_ARGV[0].nil?
+        daemon_show_usage
+        start = false
+    end
+  end
+ 
+  start
+end
+
+
+################# MAIN #####################
+daemon_start if daemon_parse_opts

metadata

@@ -0,0 +1,187 @@
+--- !ruby/object:Gem::Specification
+name: queue_dispatcher
+version: !ruby/object:Gem::Version
+  version: 1.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Philip Kurmann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sys-proctable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: deadlock_retry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: spawn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: haml
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: will_paginate
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jquery-rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: The QueueDispatcher let you execute queues as workers.
+email:
+- philip.kurmann@inwork.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.rdoc
+- Rakefile
+- app/assets/images/icon_acquire_lock.png
+- app/assets/images/icon_error.png
+- app/assets/images/icon_init_queue.gif
+- app/assets/images/icon_pending.gif
+- app/assets/images/icon_running.gif
+- app/assets/images/icon_successful.png
+- app/assets/images/icon_warning.gif
+- app/assets/images/lock.png
+- app/assets/javascript/tasks.js.coffee
+- app/helpers/tasks_helper.rb
+- app/models/task_dependency.rb
+- app/views/queue_dispatcher_views/_search_results_events.html.haml
+- app/views/queue_dispatcher_views/_search_results_my_events.html.haml
+- app/views/queue_dispatcher_views/_task_event.html.haml
+- app/views/queue_dispatcher_views/_task_footer.html.erb
+- app/views/queue_dispatcher_views/_task_header.html.erb
+- app/views/queue_dispatcher_views/_task_my_event.html.haml
+- app/views/queue_dispatcher_views/expand_event.js.rjs
+- app/views/queue_dispatcher_views/my_events.html.haml
+- app/views/queue_dispatcher_views/my_events.js.erb
+- app/views/queue_dispatcher_views/update_events.js.erb
+- lib/generators/queue_dispatcher/migration/migration_generator.rb
+- lib/generators/queue_dispatcher/migration/templates/task_dependencies.rb
+- lib/generators/queue_dispatcher/migration/templates/task_queues.rb
+- lib/generators/queue_dispatcher/migration/templates/tasks.rb
+- lib/queue_dispatcher.rb
+- lib/queue_dispatcher/acts_as_task.rb
+- lib/queue_dispatcher/acts_as_task_controller.rb
+- lib/queue_dispatcher/acts_as_task_queue.rb
+- lib/queue_dispatcher/deserialization_error.rb
+- lib/queue_dispatcher/message_sending.rb
+- lib/queue_dispatcher/psych_ext.rb
+- lib/queue_dispatcher/qd_logger.rb
+- lib/queue_dispatcher/rc_and_msg.rb
+- lib/queue_dispatcher/serialization/active_record.rb
+- lib/queue_dispatcher/syck_ext.rb
+- lib/queue_dispatcher/version.rb
+- lib/queue_dispatcher/yaml_ext.rb
+- lib/tasks/queue_dispatcher.rake
+- queue_dispatcher.gemspec
+- script/queue_worker_dispatcher
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: This Gem installs a queue dispatcher for handling asynchronous tasks
+test_files: []