afterparty 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 !binary "U0hBMQ==":
-  metadata.gz: d5a18d9460144527504869b72c7a377ce2c6eda0
-  data.tar.gz: 4d9b52a88c5fea21d37da73b1a13f5f5bf180c9e
+  metadata.gz: b61c1a12ac1be3fb98d8615c4126689032529792
+  data.tar.gz: 9ee9c0fd733e0d96133040e956a021e0f3f5c356
 !binary "U0hBNTEy":
-  metadata.gz: d6323eb6eb7a72b1090207d498ae02890dbfca347bdc37414453f67441725e966b4205651d53d4756e15ce7a786ff1a1058c98572439c67e931ab0f150ce70c7
-  data.tar.gz: 1888cd7d556ee3f5164935ac9fb7a9b57dcf9fb1808ff8eae57315bd8175f3080e2a1dca599437bbf87770f11acc48b598fd62a1272a4c8077f0c7385304d9a1
+  metadata.gz: 6d4e7ebafeff397b8aa751c60e8e1198a435701984be74d7f5275be86b55ac5cc41512db4846a42a81abecf57a5a56f9770554b39ef5fc9380570fb053e2e3f1
+  data.tar.gz: c744c87dc122d7901d6ebcc9b758740c6c563aa01ff8a802e23a7f49aa68d1b198bafe2460da9ba826efd3640cefde72e8e5f047d7139ad22a16f670c919dffe

data/.travis.yml

@@ -3,4 +3,6 @@ rvm:
   - 1.9.3
   - 2.0.0
 services:
-  - redis-server
+  - redis-server
+env:
+  - AFTERPARTY_JOB_TIME=15 AFTERPARTY_SLOW_TIME=50

data/README.md

@@ -1,6 +1,8 @@
 # Afterparty
 
-A Rails 4 compatible queue with support for executing jobs in the future and serialization with Redis.
+[![Build Status](https://travis-ci.org/hstove/afterparty.png?branch=master)](https://travis-ci.org/hstove/afterparty)
+
+A Rails 4 compatible queue with support for executing jobs in the future and persistence with Redis.
 
 ## Installation
 
@@ -41,7 +43,7 @@ end
 Then add it to the queue at any time.
 
 ~~~Ruby
-Rails.queue << Job.new
+Rails.configuration.queue << Job.new
 ~~~
 
 If your job responds to an `execute_at` method, the queue will wait to process that job until the specified time.

data/afterparty.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   
   spec.add_dependency "redis"
+  spec.add_dependency "iconv"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/app/assets/javascripts/afterparty.js.coffee

@@ -0,0 +1,14 @@
+$(document).ready ->
+  # $('.debug').click (e) ->
+  #   $el = $(e.target)
+  #   $tr = $el.parents('tr')
+  #   $tr.next().toggle()
+  #   $tr.toggleClass('debugged')
+  #   false
+  $('.job-row').click (e) ->
+    $el = $(e.target)
+    return true if $el.hasClass('job-action')
+    $tr = $el.parents('tr')
+    $tr.next().toggle()
+    $tr.toggleClass('debugged')
+    false

data/app/assets/stylesheets/afterparty.css.sass

@@ -0,0 +1,85 @@
+body
+  :font-family helvetica
+  :background-color whitesmoke
+  :color lighten(black,30%)
+  :margin 0 20px
+
+a:visited, a
+  :color rgb(96, 89, 180)
+  :text-decoration none
+
+span
+  :font-size 12px
+
+.job-table
+  :border-spacing 0
+  :max-width 100%
+  :margin 0px auto
+  thead tr:first-child
+    th
+      :border-width 1px 1px 1px 0
+      &:first-child
+        :border-width 1px 1px 1px 1px
+    th:first-child
+      :border-top-left-radius 2px
+    th:last-child
+      :border-top-right-radius 2px
+  tbody tr:last-child
+    td:first-child
+      :border-bottom-left-radius 2px
+    td:last-child
+      :border-bottom-right-radius 2px
+  tr
+    td, th
+      :border 1px solid #ccc
+      :border-width 0 1px 1px 0
+      :background-color lighten(whitesmoke, 5%)
+      :padding 12px 5px
+      :text-align left
+      :vertical-align middle
+      :word-wrap break-word
+      &:nth-child(2)
+        :width 70%
+      &:first-child
+        :border-width 0 1px 1px 1px
+      &:first-child, &:last-child
+        :width 15% !important
+    &.debugged
+      td
+        :background-color rgba(255,255,0,0.25)
+    &.job-row
+      :cursor pointer
+      h3
+        :margin 0
+      &:hover
+        td
+          :background-color rgba(0,0,0,0.15)
+        .description
+          :color white
+  .debug-row
+    :display none
+    :max-width 100%
+    td pre
+      :word-wrap break-word
+      :white-space pre-wrap 
+
+.distance-past, .distance-future
+  :font-size 12px
+.distance-future
+  :color darken(rgb(0,255,0), 20%)
+.distance-past
+  :color rgba(255,0,0,0.6)
+
+.description
+  :font-size 12px
+  :color #bbb
+  :margin-left 10px
+
+.job-action
+  :text-decoration underline
+  :font-size 12px
+  :margin 0 2px
+
+.job_id
+  :color #bbb
+  :font-weight 700

data/app/controllers/afterparty/dashboard_controller.rb

@@ -0,0 +1,20 @@
+module Afterparty
+  class DashboardController < ApplicationController
+    layout false
+    def index
+      @queues = Afterparty.queues
+      if params[:completed]
+        @jobs = queue.completed_with_scores
+      else
+        @jobs = queue.jobs_with_scores
+      end
+    end
+
+    def queue
+      Rails.configuration.queue
+    end
+
+    def run
+    end
+  end
+end

data/app/views/afterparty/dashboard/index.html.haml

@@ -0,0 +1,66 @@
+!!!
+%html
+
+  %head
+    %title== #{Rails.application.class.parent_name} Job Queue
+    %meta{"http-equiv"=>"Content-Type", :content=>"text/html; charset=utf-8"}
+    / %meta{name: "viewport", content: "width=device-width, initial-scale=1.0"}
+    = stylesheet_link_tag "afterparty"
+    = javascript_include_tag "jquery", "afterparty"
+    = csrf_meta_tag
+    = favicon_link_tag
+    = yield(:head)
+
+  %body
+    %h1
+      Viewing
+      - if params[:completed]
+        = pluralize @jobs.size, "completed job"
+        %h3= link_to "View Job Queue", afterparty_engine.dashboard_path
+      - else
+        = pluralize @jobs.size, "job"
+        %h3= link_to "View Completed Jobs", afterparty_engine.dashboard_path(completed: true)
+    = @queues
+    %table.job-table
+      %thead
+        %tr
+          %th Execute At
+          / %th Job
+          / %th Actions
+      %tbody
+        - if @jobs.empty?
+          %tr
+            %td{colspan: 3}
+              %em No jobs to show...
+        - else
+          - @jobs.each do |job_container|
+            %tr.job-row
+              %td
+                %h3
+                  - if job_container.job
+                    = link_to job_container.job_class, "#", class: 'debug'
+                    - if job_container.job.respond_to? :description
+                      %span.description= job_container.job.description
+                  - else
+                    %em Error marshaling job
+                =# job_container.execute_at.strftime("%B %d, %Y at %l:%M %P")
+                - distance = time_ago_in_words(job_container.execute_at)
+                - if job_container.execute_at > Time.now
+                  %span.distance-future= "in #{distance}"
+                - else
+                  %span.distance-past= "#{distance} ago"
+                - if (_id = job_container.job_id) && (_queue = job_container.queue_name)
+                  %span.job_id== ##{_id} #{_queue}
+                  - unless params[:completed]
+                    = link_to "run", afterparty_engine.run_job_path(job_id: _id, queue: _queue), class: 'job-action'
+                    = link_to "delete", afterparty_engine.delete_job_path(job_id: _id, queue: _queue), class: 'job-action'
+            %tr.debug-row
+              %td
+                - if job_container.job
+                  = debug job_container.job
+                - else
+                  = job_container.raw_string
+
+%p
+  Current Time:
+  = Time.now.strftime("%B %d, %Y at %l:%M %P")

data/config/routes.rb

@@ -0,0 +1,9 @@
+Afterparty::Engine.routes.draw do
+  get "/" => "afterparty/dashboard#index", as: :dashboard
+  get "/run" => "afterparty/dashboard#run", as: :run_job
+  get "/delete" => "afterparty/dashboard#run", as: :delete_job
+end
+
+Rails.application.routes.draw do
+  mount Afterparty::Engine, at: "afterparty", as: "afterparty_engine"
+end

data/lib/afterparty.rb

@@ -1,14 +1,63 @@
 require 'logger'
 require 'afterparty/queue_helpers'
 require 'afterparty/redis_queue'
+require 'redis'
 Dir[File.expand_path('../afterparty/*', __FILE__)].each { |f| require f }
 
 
 module Afterparty
+  @@redis = Redis.new
+
   def self.redis
     @@redis
   end
   def self.redis=(redis)
     @@redis = redis
   end
+
+  def self.clear namespace=:default
+    redis_call namespace, :del
+  end
+
+  def self.redis_call namespace, command, *args
+    @@redis.send(command, redis_queue_name(namespace), *args)
+  end
+
+  def self.redis_queue_name namespace=:default
+    "afterparty_#{namespace}_queue"
+  end
+
+  def self.queues
+    @@redis.smembers "afterparty_queues"
+  end
+
+  def self.add_queue name
+    @@redis.sadd "afterparty_queues", name
+  end
+
+  def self.next_job_id namespace=:default
+    @@redis.incr "afterparty_#{namespace.to_s}_job_id"
+  end
+
+  def self.load(raw)
+    begin
+      begin
+        job = Marshal.load(raw)
+        job = Marshal.load(job) if String === job
+        return job
+      rescue NameError => e
+        # lots of marshal load errors are because something that hasn't been
+        # required. recursively require on these errors
+        name = e.message.gsub("uninitialized constant ","").downcase
+        begin
+          require "#{name}"
+          return load(raw)
+        rescue LoadError
+        end
+      end
+    rescue
+      return nil
+    end
+  end
+
 end

data/lib/afterparty/engine.rb

@@ -0,0 +1,7 @@
+require 'rails'
+module Afterparty
+  class Engine < Rails::Engine
+    engine_name :afterparty
+
+  end
+end

data/lib/afterparty/job_container.rb

@@ -0,0 +1,36 @@
+require 'iconv'
+require 'date'
+
+module Afterparty
+  class JobContainer
+    attr_accessor :job, :raw, :execute_at, :job_id, :queue_name
+
+    #intialized from redis's WITHSCORES function
+    def initialize _raw, timestamp
+      @execute_at = Time.at(timestamp)
+      begin
+        @job = Afterparty.load(_raw)
+        @job_id = job.afterparty_job_id if @job.respond_to? :afterparty_job_id
+        @queue_name = job.afterparty_queue if @job.respond_to? :afterparty_queue
+      rescue Exception => e
+        ap "Error during load: #{e.message}"
+        @job = nil
+      end
+      @raw = _raw
+      self
+    end
+
+    def job_class
+      if @job
+        @job.class
+      else
+        nil
+      end
+    end
+
+    def raw_string
+      ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+      ic.iconv(@raw.dup + ' ')[0..-2]
+    end
+  end
+end

data/lib/afterparty/queue_helpers.rb

@@ -4,8 +4,9 @@ module Afterparty
       @temp_namespace = namespace
     end
 
-    def redis_queue_name
-      "afterparty_#{@temp_namespace || @options[:namespace]}_queue"  
+    def redis_queue_name  
+      puts (a = Afterparty.redis_queue_name(@temp_namespace || @options[:namespace]))
+      a
     end
 
     def clear
@@ -13,7 +14,7 @@ module Afterparty
     end
 
     def redis_call command, *args
-      result = Afterparty.redis.send(command, redis_queue_name, *args)
+      result = Afterparty.redis_call (@temp_namespace || @options[:namespace]), command, *args
       @temp_namespace = nil
       result
     end
@@ -31,13 +32,17 @@ module Afterparty
     end
 
     def jobs_with_scores
-      redis_call :zrange, 0, -1, {withscores: true}
+      hash_from_scores(redis_call(:zrange, 0, -1, {withscores: true}))
     end
 
     def valid_jobs
       redis_call :zrangebyscore, 0, Time.now.to_i
     end
 
+    def next_valid_job
+      valid_jobs.first
+    end
+
     def jobs_empty?
       count = total_jobs_count
       # ap count
@@ -52,8 +57,32 @@ module Afterparty
       @@redis
     end
 
+    def last_completed
+      @temp_namespace = "completed"
+      redis_call(:zrange, -1, -1).first
+    end
+
+    def completed
+      @temp_namespace = "completed"
+      redis_call(:zrange, -20, -1).reverse
+    end
+
+    def completed_with_scores
+      @temp_namespace = "completed"
+      hash_from_scores(redis_call(:zrange, -20, -1, withscores: true)).reverse
+    end
+
+
     private
 
+    def hash_from_scores raw
+      arr = []
+      raw.each do |group|
+        arr << Afterparty::JobContainer.new(group[0], group[1])
+      end
+      arr
+    end
+
     # returns true if job has an :execute_at value
     def job_valid? job
       job.respond_to?(:execute_at) && !job.execute_at.nil?

data/lib/afterparty/redis_queue.rb

@@ -4,9 +4,10 @@ module Afterparty
     include Afterparty::QueueHelpers
 
     def initialize options={}, consumer_options={}
-      @consumer = ThreadedQueueConsumer.new(self, consumer_options).start
+      # @consumer = ThreadedQueueConsumer.new(self, consumer_options).start
       @options = options
       @options[:namespace] ||= "default"
+      Afterparty.add_queue @options[:namespace]
       @options[:sleep] ||= 5
       @mutex = Mutex.new
     end
@@ -14,8 +15,13 @@ module Afterparty
     def push job
       @mutex.synchronize do
         return nil if job.nil?
+        job.class.module_eval do
+          attr_accessor :afterparty_job_id, :afterparty_queue
+        end
+        queue_name = @temp_namespace || @options[:namespace]
+        job.afterparty_queue = queue_name
+        job.afterparty_job_id = Afterparty.next_job_id queue_name
         async_redis_call{ redis_call :zadd, queue_time(job), Marshal.dump(job) }
-        @consumer.start unless @consumer.thread.alive?
         @temp_namespace = nil
       end
     end
@@ -25,16 +31,20 @@ module Afterparty
     def pop
       @mutex.synchronize do
         while true do
-          if jobs_empty?
-            @consumer.shutdown
-          elsif !(_jobs = valid_jobs).empty?
+          if !(_jobs = valid_jobs).empty?
             job_dump = _jobs[0]
             async_redis_call do
               redis_call :zrem, job_dump
               @temp_namespace = "completed"
               redis_call :zadd, Time.now.to_i, job_dump
             end
-            return Marshal.load(job_dump)
+            begin
+              return Marshal.load(job_dump)
+            rescue ArgumentException => e
+              puts "You encountered an argument exception while deserializing a job."
+              puts "Message: #{e.message}"
+              raise e
+            end
           end
           sleep(@options[:sleep])
         end

data/lib/afterparty/version.rb

@@ -1,3 +1,3 @@
 module Afterparty
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/afterparty/worker.rb

@@ -0,0 +1,56 @@
+module Afterparty
+  class Worker
+    include QueueHelpers
+
+    def initialize options = {}
+      @options = options
+      @options[:adapter] ||= :redis
+      @options[:namespace] ||= :default
+      @options[:sleep] ||= 10
+      @options[:logger] ||= Logger.new($stderr)
+      self
+    end
+
+    def consume
+      @stopped = false
+      # puts "starting worker with namespace [#{@options[:namespace]}]."
+      @thread = Thread.new {
+        consume_sync
+      }
+      @thread
+    end
+
+    def consume_sync
+      while !@stopped
+        job = next_valid_job
+        if job
+          async_redis_call do
+            @temp_namespace = "completed"
+            redis_call :zadd, Time.now.to_i, Marshal.dump(job)
+            redis_call :zrem, job
+          end
+          run job
+        else
+          sleep(@options[:sleep])
+        end
+      end
+    end
+
+    def stop
+      @stopped = true
+      @thread.join(0)
+    end
+
+    def run(job)
+      fork do
+        Marshal.load(job).run
+      end
+    rescue Exception => exception
+      handle_exception job, exception
+    end
+
+    def handle_exception(job, exception)
+      @options[:logger].error "Job Error: #{job.inspect}\n#{exception.message}\n#{exception.backtrace.join("\n")}"
+    end
+  end
+end

data/lib/tasks/tasks.rake

@@ -0,0 +1,25 @@
+namespace :jobs do
+  require 'mail'
+
+  desc "Start a new worker"
+  task work: :environment do
+    worker = Afterparty::Worker.new
+    worker.consume_sync
+  end
+
+  desc "Clear all jobs"
+  task clear: :environment do
+    Rails.configuration.queue.clear
+  end
+
+  desc "List Jobs"
+  task list: :environment do
+    jobs = Rails.configuration.queue.jobs_with_scores
+    puts "#{jobs.values.size} total jobs."
+    jobs.each do |time, job|
+      puts time
+      puts job
+    end
+  end
+end
+

data/spec/redis_queue_spec.rb

@@ -8,10 +8,16 @@ describe Afterparty::RedisQueue do
     @q = Afterparty::TestRedisQueue.new({sleep: 0.5})
   end
 
+  after do
+    @worker.stop
+  end
+
   before :each do
-    @q.completed_jobs.clear
+    @worker = Afterparty::Worker.new({sleep: 0.5})
+    @worker.consume
     @q.clear
-    Afterparty.redis.quit
+    @job_time = (ENV['AFTERPARTY_JOB_TIME'] || 3).to_i
+    @slow_job_time = (ENV['AFTERPARTY_SLOW_TIME'] || 10).to_i
   end
 
   it "pushes nil without errors" do
@@ -27,34 +33,35 @@ describe Afterparty::RedisQueue do
   it "executes the job" do
     job = TestJob.new
     @q.push(job)
-    complete.size.should eq(0)
-    chill(3)
-    complete.size.should eq(1)
+    @q.jobs.size.should eq(1)
+    chill(@job_time)
+    @q.jobs.size.should eq(0)
   end
 
   it "removes items from the queue after running them" do
     @q.push TestJob.new
-    chill(1)
-    @q.jobs.should_not include(@job)
+    chill(@job_time)
+    @q.jobs.size.should == 0
   end
 
   it "doesn't execute jobs that execute in a while" do
     job = TestJob.new
     job.execute_at = Time.now + 200
     @q.push job
-    chill(3)
-    complete.size.should eq(0)
+    chill(@job_time)
+    @q.jobs.size.should eq(1)
   end
 
   it "waits the correct amount of time to execute a job" do
     job = TestJob.new
-    job.execute_at = Time.now + 2
+    job.execute_at = Time.now + 5
     @q.push(job)
-    chill(8)
-    complete.size.should eq(1)
+    @q.jobs.size.should eq(1)
+    chill(@slow_job_time)
+    @q.jobs.size.should eq(0)
   end
 
-  it "doesn't execute the job synchronously when added" do
+  it "doesn't wait and execute the job synchronously when added" do
     job = test_job 100
     t = Time.now
     @q.push(job)
@@ -66,9 +73,9 @@ describe Afterparty::RedisQueue do
     early_job = test_job
     @q.push(late_job)
     @q.push(early_job)
-    chill(3)
-    complete.size.should eq(1)
-    complete[0].execute_at.should be(nil)
+    chill(@job_time)
+    (jobs = @q.jobs).size.should eq(1)
+    jobs[0].execute_at.should_not be(nil)
   end
 
   class ErrorJob

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: afterparty
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Hank Stoever
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-14 00:00:00.000000000 Z
+date: 2013-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -24,6 +24,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: iconv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -68,12 +82,21 @@ files:
 - README.md
 - Rakefile
 - afterparty.gemspec
+- app/assets/javascripts/afterparty.js.coffee
+- app/assets/stylesheets/afterparty.css.sass
+- app/controllers/afterparty/dashboard_controller.rb
+- app/views/afterparty/dashboard/index.html.haml
+- config/routes.rb
 - dump.rdb
 - lib/afterparty.rb
+- lib/afterparty/engine.rb
+- lib/afterparty/job_container.rb
 - lib/afterparty/queue_helpers.rb
 - lib/afterparty/redis_queue.rb
 - lib/afterparty/threaded_queue_consumer.rb
 - lib/afterparty/version.rb
+- lib/afterparty/worker.rb
+- lib/tasks/tasks.rake
 - spec/helpers.rb
 - spec/redis_queue_spec.rb
 - spec/spec_helper.rb