rabbit_jobs 0.0.4 → 0.0.5

data/README.md

@@ -1,6 +1,6 @@
 # RabbitJobs
 
-TODO: Write a gem description
+Schedule jobs on rabbitmq and run them in background.
 
 ## Installation
 
@@ -18,7 +18,7 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+TODO: Write usage instructions here.
 
 ## Contributing
 

data/examples/configuration.rb

@@ -14,7 +14,6 @@ RabbitJobs.configure do |c|
 end
 
 puts JSON.pretty_generate(RabbitJobs.config.to_hash)
-
 puts JSON.pretty_generate(RabbitJobs.config.queues)
 
 class MyJob < RabbitJobs::Job

data/lib/rabbit_jobs.rb

@@ -10,6 +10,7 @@ require 'rabbit_jobs/logger'
 require 'rabbit_jobs/job'
 require 'rabbit_jobs/publisher'
 require 'rabbit_jobs/worker'
+require 'rabbit_jobs/scheduler'
 
 module RabbitJobs
   extend self
@@ -21,4 +22,8 @@ module RabbitJobs
   def publish_to(routing_key, klass, opts = {}, *params)
     RabbitJobs::Publisher.publish_to(routing_key, klass, opts, *params)
   end
+end
+
+module RJ
+  include RabbitJobs
 end

data/lib/rabbit_jobs/amqp_helpers.rb

@@ -4,7 +4,6 @@ module RabbitJobs
   module AmqpHelpers
 
     # Calls given block with initialized amqp
-
     def amqp_with_exchange(&block)
       raise ArgumentError unless block
 
@@ -23,6 +22,23 @@ module RabbitJobs
       end
     end
 
+    def em_amqp_with_exchange(&block)
+      raise ArgumentError unless block
+
+      connection = AMQP.connect(host: RabbitJobs.config.host)
+      channel = AMQP::Channel.new(connection)
+
+      channel.on_error do |ch, channel_close|
+        puts "Channel-level error: #{channel_close.reply_text}, shutting down..."
+        connection.close { EM.stop }
+      end
+
+      exchange = channel.direct(RabbitJobs.config[:exchange], RabbitJobs.config[:exchange_params])
+
+      # go work
+      block.call(connection, exchange)
+    end
+
     def amqp_with_queue(routing_key, &block)
 
       raise ArgumentError unless routing_key && block

data/lib/rabbit_jobs/job.rb

@@ -21,7 +21,7 @@ module RabbitJobs::Job
     def run_perform
       if @child_pid = fork
         srand # Reseeding
-        log "Forked #{@child_pid} at #{Time.now} to process #{self.class}.perform(#{ params.map(&:inspect).join(', ') })"
+        RabbitJobs::Logger.log "Forked #{@child_pid} at #{Time.now} to process #{self.class}.perform(#{ params.map(&:inspect).join(', ') })"
         Process.wait(@child_pid)
         yield if block_given?
       else

data/lib/rabbit_jobs/publisher.rb

@@ -21,7 +21,11 @@ module RabbitJobs
       job = klass.new(*params)
       job.opts = opts
 
-      publish_job_to(routing_key, job)
+      if defined?(EM) && EM.reactor_running?
+        em_publish_job_to(routing_key, job)
+      else
+        publish_job_to(routing_key, job)
+      end
     end
 
     def publish_job_to(routing_key, job)
@@ -33,7 +37,24 @@ module RabbitJobs
 
         payload = job.payload
         exchange.publish(job.payload, Configuration::DEFAULT_MESSAGE_PARAMS.merge({routing_key: routing_key})) {
-          connection.close { EM.stop }
+          connection.close {
+            EM.stop
+          }
+        }
+      end
+    end
+
+    def em_publish_job_to(routing_key, job)
+      em_amqp_with_exchange do |connection, exchange|
+
+        queue = make_queue(exchange, routing_key)
+
+        job.opts['created_at'] = Time.now.to_s
+
+        payload = job.payload
+        exchange.publish(job.payload, Configuration::DEFAULT_MESSAGE_PARAMS.merge({routing_key: routing_key})) {
+          connection.close {
+          }
         }
       end
     end

data/lib/rabbit_jobs/scheduler.rb

@@ -0,0 +1,160 @@
+# -*- encoding : utf-8 -*-
+
+require 'rufus/scheduler'
+require 'thwait'
+require 'yaml'
+
+module RabbitJobs
+  class Scheduler
+    include AmqpHelpers
+    include Logger
+
+    attr_accessor :pidfile, :background, :schedule
+
+    def load_default_schedule
+      if defined?(Rails)
+        file = Rails.root.join('config/schedule.yml')
+        if file.file?
+          @schedule = YAML.load_file(file)
+        end
+      end
+    end
+
+    # Pulls the schedule from Resque.schedule and loads it into the
+    # rufus scheduler instance
+    def load_schedule!
+      @schedule ||= load_default_schedule
+
+      raise "You should setup a schedule or place it in config/schedule.yml" unless schedule
+
+      schedule.each do |name, config|
+        # If rails_env is set in the config, enforce ENV['RAILS_ENV'] as
+        # required for the jobs to be scheduled.  If rails_env is missing, the
+        # job should be scheduled regardless of what ENV['RAILS_ENV'] is set
+        # to.
+        if config['rails_env'].nil? || rails_env_matches?(config)
+          log! "Scheduling #{name} "
+          interval_defined = false
+          interval_types = %w{cron every}
+          interval_types.each do |interval_type|
+            if !config[interval_type].nil? && config[interval_type].length > 0
+              rufus_scheduler.send(interval_type, config[interval_type]) do
+                log! "queueing #{config['class']} (#{name})"
+                publish_from_config(config)
+              end
+              interval_defined = true
+              break
+            end
+          end
+          unless interval_defined
+            log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
+          end
+        end
+      end
+    end
+
+    # Returns true if the given schedule config hash matches the current ENV['RAILS_ENV']
+    def rails_env_matches?(config)
+      config['rails_env'] && ENV['RAILS_ENV'] && config['rails_env'].gsub(/\s/,'').split(',').include?(ENV['RAILS_ENV'])
+    end
+
+    # Publish a job based on a config hash
+    def publish_from_config(config)
+      args = config['args'] || config[:args] || []
+      klass_name = config['class'] || config[:class]
+      params = args.is_a?(Hash) ? [args] : Array(args)
+      queue = config['queue'] || config[:queue] || RabbitJobs.config.routing_keys.first
+
+      log "publishing #{config}"
+      RabbitJobs.publish_to(queue, klass_name.constantize, nil, *params)
+    rescue
+      log! "Failed to enqueue #{klass_name}:\n #{$!}\n params = #{params.inspect}"
+    end
+
+    def rufus_scheduler
+      @rufus_scheduler ||= Rufus::Scheduler.start_new
+    end
+
+    # Stops old rufus scheduler and creates a new one.  Returns the new
+    # rufus scheduler
+    def clear_schedule!
+      rufus_scheduler.stop
+      @rufus_scheduler = nil
+      rufus_scheduler
+    end
+
+    # Subscribes to channel and working on jobs
+    def work(time = 0)
+      startup
+
+      processed_count = 0
+      amqp_with_exchange do |connection, exchange|
+        load_schedule!
+
+        check_shutdown = Proc.new {
+          if @shutdown
+            connection.close {
+              File.delete(self.pidfile) if self.pidfile
+              EM.stop { exit! }
+            }
+          end
+        }
+
+        if time > 0
+          # for debugging
+          EM.add_timer(time) do
+            log "Stopping scheduler..."
+            self.shutdown
+          end
+        end
+
+        log "Scheduler started."
+
+        EM.add_periodic_timer(1) do
+          check_shutdown.call
+        end
+      end
+    end
+
+    def shutdown
+      log "Stopping scheduler..."
+      @shutdown = true
+    end
+
+    def startup
+      # prune_dead_workers
+
+      Process.daemon(true) if self.background
+
+      if self.pidfile
+        File.open(self.pidfile, 'w') { |f| f << Process.pid }
+      end
+
+      # Fix buffering so we can `rake rj:work > resque.log` and
+      # get output from the child in there.
+      $stdout.sync = true
+
+      @shutdown = false
+
+      Signal.trap('TERM') { shutdown }
+      Signal.trap('INT')  { shutdown! }
+    end
+
+    def shutdown!
+      shutdown
+      kill_child
+    end
+
+    def kill_child
+      if @job && @job.child_pid
+        # log! "Killing child at #{@child}"
+        if Kernel.system("ps -o pid,state -p #{@job.child_pid}")
+          Process.kill("KILL", @job.child_pid) rescue nil
+        else
+          # log! "Child #{@child} not found, restarting."
+          # shutdown
+        end
+      end
+    end
+  end
+end

data/lib/rabbit_jobs/tasks.rb

@@ -1,11 +1,13 @@
 # require 'resque/tasks'
 # will give you the resque tasks
 
+require 'rabbit_jobs'
+
 namespace :rj do
   task :setup
 
   desc "Start a Rabbit Jobs worker"
-  task :work => [ :preload, :setup ] do
+  task :worker => [ :preload, :setup ] do
     require 'rabbit_jobs'
 
     queues = (ENV['QUEUES'] || ENV['QUEUE']).to_s.split(',')
@@ -18,10 +20,23 @@ namespace :rj do
       # worker.very_verbose = ENV['VVERBOSE']
     end
 
-    # worker.log "Starting worker #{worker.pid}"
-    # worker.verbose = true
-    worker.work 10
-    # worker.work(ENV['INTERVAL'] || 5) # interval, will block
+    worker.work
+  end
+
+  desc "Start a Rabbit Jobs scheduler"
+  task :scheduler => [ :preload, :setup ] do
+
+    queues = (ENV['SCHEDULE']).to_s.split(',')
+
+    begin
+      scheduler = RabbitJobs::Scheduler.new(*queues)
+      scheduler.pidfile = ENV['PIDFILE']
+      scheduler.background = %w(yes true).include? ENV['BACKGROUND']
+      RabbitJobs::Logger.verbose = true if ENV['VERBOSE']
+      # worker.very_verbose = ENV['VVERBOSE']
+    end
+
+    scheduler.work
   end
 
   desc "Start multiple Resque workers. Should only be used in dev mode."
@@ -30,7 +45,7 @@ namespace :rj do
 
     ENV['COUNT'].to_i.times do
       threads << Thread.new do
-        system "rake resque:work"
+        system "rake resque:worker"
       end
     end
 
@@ -41,7 +56,8 @@ namespace :rj do
   task :preload => :setup do
     if defined?(Rails) && Rails.respond_to?(:application)
       # Rails 3
-      Rails.application.eager_load!
+      # Rails.application.eager_load!
+      Rails.application.require_environment!
     elsif defined?(Rails::Initializer)
       # Rails 2.3
       $rails_rake_task = false

data/lib/rabbit_jobs/version.rb

@@ -1,3 +1,3 @@
 module RabbitJobs
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/tasks/rabbit_jobs.rake

@@ -1,3 +1,3 @@
-
 $LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+
 require 'rabbit_jobs/tasks'

data/rabbit_jobs.gemspec

@@ -19,4 +19,5 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "amqp", "~> 0.9"
   gem.add_dependency "rake"
+  gem.add_dependency "rufus-scheduler", "~> 2.0"
 end

data/spec/fixtures/schedule.yml

@@ -0,0 +1,13 @@
+do_a_job:
+  cron: "* * * * *" # run every minute
+  queue: queue_name
+  class: MyJobClass
+  args: "string to be passed to MyJobClass.perform"
+  description: "Description is not supported yet"
+
+another_job:
+  every: 1h # run every 1 hour
+  queue: queue_name
+  class: MyJobClass
+  args: "string to be passed to MyJobClass.perform"
+  description: "Description is not supported yet"

data/spec/integration/scheduler_spec.rb

@@ -0,0 +1,17 @@
+# -*- encoding : utf-8 -*-
+require 'spec_helper'
+
+describe RabbitJobs::Scheduler do
+  it 'should start with config.yml' do
+    scheduler = RabbitJobs::Scheduler.new
+    scheduler.schedule = YAML.load_file(File.expand_path('../../fixtures/schedule.yml', __FILE__))
+
+    scheduler.pidfile = '/tmp/rj_scheduler.pid'
+    scheduler.background = false
+
+    scheduler.work(10) # work for 1 second
+
+    puts "messages queued: " + RabbitJobs::Publisher.purge_queue('default').to_s
+    RabbitJobs::Publisher.purge_queue('default').should == 0
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rabbit_jobs
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-31 00:00:00.000000000 Z
+date: 2012-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &70340051105620 !ruby/object:Gem::Requirement
+  requirement: &70230061017880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.9'
   type: :runtime
   prerelease: false
-  version_requirements: *70340051105620
+  version_requirements: *70230061017880
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70340051105000 !ruby/object:Gem::Requirement
+  requirement: &70230061016460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70340051105000
+  version_requirements: *70230061016460
+- !ruby/object:Gem::Dependency
+  name: rufus-scheduler
+  requirement: &70230061008760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70230061008760
 description: Background jobs on RabbitMQ
 email:
 - lazureykis@gmail.com
@@ -54,6 +65,7 @@ files:
 - lib/rabbit_jobs/job.rb
 - lib/rabbit_jobs/logger.rb
 - lib/rabbit_jobs/publisher.rb
+- lib/rabbit_jobs/scheduler.rb
 - lib/rabbit_jobs/tasks.rb
 - lib/rabbit_jobs/version.rb
 - lib/rabbit_jobs/worker.rb
@@ -61,7 +73,9 @@ files:
 - rabbit_jobs.gemspec
 - spec/fixtures/config.yml
 - spec/fixtures/jobs.rb
+- spec/fixtures/schedule.yml
 - spec/integration/publisher_spec.rb
+- spec/integration/scheduler_spec.rb
 - spec/integration/worker_spec.rb
 - spec/spec_helper.rb
 - spec/unit/configuration_spec.rb
@@ -89,14 +103,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Background jobs on RabbitMQ
 test_files:
 - spec/fixtures/config.yml
 - spec/fixtures/jobs.rb
+- spec/fixtures/schedule.yml
 - spec/integration/publisher_spec.rb
+- spec/integration/scheduler_spec.rb
 - spec/integration/worker_spec.rb
 - spec/spec_helper.rb
 - spec/unit/configuration_spec.rb