afterparty 0.0.21 → 0.1.0

data/lib/afterparty/afterparty_job.rb

@@ -0,0 +1,31 @@
+require 'active_record'
+require 'afterparty/queue_helpers'
+class AfterpartyJob < ::ActiveRecord::Base
+  # include Afterparty::QueueHelpers
+
+  validates_presence_of :job_dump, :execute_at, :queue
+
+  scope :incomplete, -> { where(completed: false).order("execute_at") }
+  scope :valid, -> { incomplete.where(execute_at: 10.years.ago..DateTime.now) }
+  scope :completed, -> { where(completed: true).order("execute_at desc") }
+
+  def self.make_with_job job, queue=:default
+    afterparty_job = AfterpartyJob.new
+    afterparty_job.job_dump = job.to_yaml
+    afterparty_job.execute_at = Afterparty.queue_time(job)
+    afterparty_job.queue = queue
+    afterparty_job.completed = false
+    afterparty_job.save
+    afterparty_job
+  end
+
+  def reify
+    Afterparty.load(job_dump)
+  end
+
+  def execute
+    if (j = reify)
+      j.run
+    end
+  end
+end

data/lib/afterparty/engine.rb

@@ -0,0 +1,7 @@
+require 'rails'
+module Afterparty
+  class Engine < Rails::Engine
+    engine_name :afterparty
+
+  end
+end

data/lib/afterparty/job_container.rb

@@ -0,0 +1,36 @@
+require 'iconv'
+require 'date'
+
+module Afterparty
+  class JobContainer
+    attr_accessor :job, :raw, :execute_at, :job_id, :queue_name
+
+    #intialized from redis's WITHSCORES function
+    def initialize _raw, timestamp
+      @execute_at = Time.at(timestamp)
+      begin
+        @job = Afterparty.load(_raw)
+        @job_id = job.afterparty_job_id if @job.respond_to? :afterparty_job_id
+        @queue_name = job.afterparty_queue if @job.respond_to? :afterparty_queue
+      rescue Exception => e
+        ap "Error during load: #{e.message}"
+        @job = nil
+      end
+      @raw = _raw
+      self
+    end
+
+    def job_class
+      if @job
+        @job.class
+      else
+        nil
+      end
+    end
+
+    def raw_string
+      ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+      ic.iconv(@raw.dup + ' ')[0..-2]
+    end
+  end
+end

data/lib/afterparty/jobs.rb

@@ -0,0 +1,41 @@
+module Afterparty
+  class MailerJob
+    attr_accessor :execute_at, :mail, :clazz, :method, :args
+    def initialize clazz, method, *args
+      # @mail = UserMailer.welcome_email(User.find(1))
+      @clazz = UserMailer
+      @method = method
+      @args = args
+    end
+    
+    def run
+      @mail = @clazz.send @method, *@args
+      @mail.deliver
+    end
+
+    def description
+      desc = "Mailer: #{(@clazz || "nil")}."
+      desc << "Method: #{(@method || nil)}."
+      desc << "Args: #{(@args || nil)}"
+    end
+  end
+
+  class BasicJob
+    attr_accessor :object, :method, :args
+    def initialize object, method, *args
+      @object = object
+      @method = method
+      @args = args
+    end
+
+    def run
+      @object.send(:method, *@args)
+    end
+
+    def description
+      desc = "Object: #{(@object || "nil")}."
+      desc << "Method: #{(@method || nil)}."
+      desc << "Args: #{(@args || nil)}"
+    end
+  end
+end

data/lib/afterparty/queue.rb

@@ -0,0 +1,53 @@
+module Afterparty
+  class Queue
+    attr_accessor :options, :temp_namespace, :login_block
+    include Afterparty::QueueHelpers
+
+    def initialize options={}, consumer_options={}
+      # @consumer = ThreadedQueueConsumer.new(self, consumer_options).start
+      @options = options
+      @options[:namespace] ||= "default"
+      # Afterparty.add_queue @options[:namespace]
+      @options[:sleep] ||= 5
+      @mutex = Mutex.new
+      @options[:logger] ||= Logger.new($stderr)
+    end
+
+    def push job
+      # @mutex.synchronize do
+        return nil if job.nil?
+        queue_name = @temp_namespace || @options[:namespace]
+        AfterpartyJob.make_with_job job, queue_name
+      # end
+    end
+    alias :<< :push
+    alias :eng :push
+
+    def pop
+      # @mutex.synchronize do
+        while true do
+          unless (_job = AfterpartyJob.valid.first).nil?
+            ap "poppin job"
+            _job.completed = true
+            _job.save
+            return _job
+          end
+          sleep(@options[:sleep])
+        end
+      # end
+    end
+  end
+  
+  class TestQueue < Queue
+    attr_accessor :completed_jobs
+    
+    def initialize opts={}, consumer_opts={}
+      super
+      @completed_jobs = []
+      @exceptions = []
+    end
+    def handle_exception job, exception
+      @exceptions << [job, exception]
+    end
+  end
+end

data/lib/afterparty/queue_helpers.rb

@@ -4,16 +4,18 @@ module Afterparty
       @temp_namespace = namespace
     end
 
-    def redis_queue_name
-      "afterparty_#{@temp_namespace || @options[:namespace]}_queue"  
+    def redis_queue_name  
+      puts (a = Afterparty.redis_queue_name(@temp_namespace || @options[:namespace]))
+      a
     end
 
     def clear
-      redis_call :del
+      # redis_call :del
+      AfterpartyJob.destroy_all
     end
 
     def redis_call command, *args
-      result = Afterparty.redis.send(command, redis_queue_name, *args)
+      result = Afterparty.redis_call (@temp_namespace || @options[:namespace]), command, *args
       @temp_namespace = nil
       result
     end
@@ -23,37 +25,110 @@ module Afterparty
     end
 
     def jobs
-      _jobs = redis_call(:zrange, 0, -1)
-      _jobs.each_with_index do |job, i|
-        _jobs[i] = Marshal.load(job)
-      end
-      _jobs
+      # _jobs = redis_call(:zrange, 0, -1)
+      # _jobs.each_with_index do |job, i|
+      #   _jobs[i] = Marshal.load(job)
+      # end
+      # _jobs
+      AfterpartyJob.incomplete
     end
 
     def jobs_with_scores
-      redis_call :zrange, 0, -1, {withscores: true}
+      hash_from_scores(redis_call(:zrange, 0, -1, {withscores: true}))
     end
 
     def valid_jobs
-      redis_call :zrangebyscore, 0, Time.now.to_i
+      # redis_call :zrangebyscore, 0, Time.now.to_i
+      AfterpartyJob.valid
+    end
+
+    def next_valid_job
+      # valid_jobs.first
+      AfterpartyJob.valid.first
     end
 
     def jobs_empty?
-      count = total_jobs_count
-      # ap count
-      count == 0
+      # count = total_jobs_count
+      # # ap count
+      # count == 0
+      AfterpartyJob.valid.empty?
     end
 
     def total_jobs_count
-      redis_call(:zcount, "-inf", "+inf")
+      # redis_call(:zcount, "-inf", "+inf")
+      AfterpartyJob.incomplete.count
     end
 
     def redis
       @@redis
     end
 
+    def last_completed
+      # @temp_namespace = "completed"
+      # redis_call(:zrange, -1, -1).first
+      AfterpartyJob.completed.first
+    end
+
+    def completed
+      # @temp_namespace = "completed"
+      # redis_call(:zrange, -20, -1).reverse
+      AfterpartyJob.completed
+    end
+
+    def completed_with_scores
+      @temp_namespace = "completed"
+      hash_from_scores(redis_call(:zrange, -20, -1, withscores: true)).reverse
+    end
+
+    def run(job)
+      real_job = job.reify
+      if real_job
+        job.execute
+      else
+        job.has_error = true
+        job.error_message = "Error marshaling job."
+      end
+      job.completed = true
+      job.completed_at = DateTime.now
+      job.save
+    rescue Exception => exception
+      handle_exception job, exception
+    end
+
+    def handle_exception(job, exception)
+      job.completed = true
+      job.completed_at = DateTime.now
+      job.has_error = true
+      job.error_message = exception.message
+      job.error_backtrace = exception.backtrace.join("\n")
+      job.save
+      logger_message = "Job Error: #{job.inspect}\n#{exception.message}"
+      logger_message << "\n#{exception.backtrace.join("\n")}"
+      @options[:logger].error logger_message
+    end
+
+    # &block takes a 'username' and 'password'
+    # argument. return true or false
+    def config_login &block
+      @login_block = block
+    end
+
+    def authenticate username, password
+      raise 'Must set queue.config_login to use dashboard' if @login_block.nil?
+      @login_block.call(username, password)
+    end
+
+
     private
 
+    def hash_from_scores raw
+      arr = []
+      raw.each do |group|
+        arr << Afterparty::JobContainer.new(group[0], group[1])
+      end
+      arr
+    end
+
     # returns true if job has an :execute_at value
     def job_valid? job
       job.respond_to?(:execute_at) && !job.execute_at.nil?

data/lib/afterparty/version.rb

@@ -1,3 +1,3 @@
 module Afterparty
-  VERSION = "0.0.21"
+  VERSION = "0.1.0"
 end

data/lib/afterparty/worker.rb

@@ -0,0 +1,46 @@
+module Afterparty
+  class Worker
+    include QueueHelpers
+
+    def initialize options = {}
+      @options = options
+      @options[:adapter] ||= :redis
+      @options[:namespace] ||= :default
+      @options[:sleep] ||= 10
+      @options[:logger] ||= Logger.new($stderr)
+      self
+    end
+
+    def consume
+      @stopped = false
+      # puts "starting worker with namespace [#{@options[:namespace]}]."
+      @thread = Thread.new {
+        consume_sync
+      }
+      @thread
+    end
+
+    def consume_next
+      if (job = next_valid_job)
+        run job
+      end
+    end
+
+    def consume_sync
+      while !@stopped
+        job = next_valid_job
+        if job
+          puts "Executing job: #{job.id}"
+          run job
+        else
+          sleep(@options[:sleep])
+        end
+      end
+    end
+
+    def stop
+      @stopped = true
+      @thread.join(0) if @thread
+    end
+  end
+end

data/lib/generators/afterparty_generator.rb

@@ -0,0 +1,9 @@
+require 'rails/generators'
+class AfterpartyGenerator < Rails::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+
+  def install
+    copy_file "jobs_migration.rb", "db/migrate/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_afterparty_jobs.rb"
+    copy_file "initializer.rb", "config/initializers/afterparty.rb"
+  end
+end

data/lib/generators/templates/initializer.rb

@@ -0,0 +1,6 @@
+queue = Rails.configuration.queue = Afterparty::Queue.new
+
+queue.config_login do |username, password|
+  # change this to something more secure!
+  user == "admin" && password == "password"
+end

data/lib/generators/templates/jobs_migration.rb

@@ -0,0 +1,21 @@
+class CreateAfterpartyJobs < ActiveRecord::Migration
+  def self.up
+    create_table :afterparty_jobs do |t|
+      t.text :job_dump
+      t.string :queue
+      t.datetime :execute_at
+      t.boolean :completed
+      t.boolean :has_error
+      t.text :error_message
+      t.text :error_backtrace
+      t.datetime :completed_at
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :afterparty_jobs
+  end
+
+end

data/lib/tasks/tasks.rake

@@ -0,0 +1,25 @@
+namespace :jobs do
+  require 'mail'
+
+  desc "Start a new worker"
+  task work: :environment do
+    worker = Afterparty::Worker.new
+    worker.consume_sync
+  end
+
+  # desc "Clear all jobs"
+  # task clear: :environment do
+  #   Rails.configuration.queue.clear
+  # end
+
+  desc "List Jobs"
+  task list: :environment do
+    jobs = Rails.configuration.queue.jobs
+    puts "#{jobs.size} total jobs."
+    jobs.each do |time, job|
+      puts time
+      puts job
+    end
+  end
+end
+

data/spec/afterparty_job_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+describe AfterpartyJob do
+  before :each do
+    AfterpartyJob.destroy_all
+  end
+
+  it "makes a job correctly" do
+    tester = test_job
+    tester.execute_at = Time.now + 10
+    job = AfterpartyJob.make_with_job tester
+    job.reload
+    (reloaded = job.reify).class.should == tester.class
+    reloaded.execute_at.should == tester.execute_at
+    job.execute_at.should == reloaded.execute_at
+  end
+
+end