resque-mongo-groups 0.5

data/.bundle/config

@@ -0,0 +1,2 @@
+--- 
+BUNDLE_DISABLE_SHARED_GEMS: "1"

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Gheorghita Catalin Bordianu
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler/setup'
+Bundler::GemHelper.install_tasks
+
+$LOAD_PATH.unshift 'lib'
+
+Dir['tasks/*.rake'].each do |f|
+  load File.expand_path(f)
+end
+
+task :default => :test
+
+desc "Run tests"
+task :test do
+  Dir['test/*_test.rb'].each do |f|
+    require File.expand_path(f)
+  end
+end

data/lib/resque_groups.rb

@@ -0,0 +1,129 @@
+require 'rubygems'
+gem 'resque-mongo'
+require 'resque'
+require 'resque/server'
+
+module Resque
+  module Plugins
+  end
+end
+
+require 'resque_groups/tracked_job'
+require 'resque_groups/server'
+
+module Resque::Plugins::Groups
+  
+  def job_groups
+    self.mongo ||= ENV['MONGO'] || 'localhost:27017'
+    @job_groups ||= @db.collection('job_groups')
+  end
+  
+  # Triggered whenever a job belonging to group 'gid' has been enqueued. If there's no entry for that group in the 'job_groups'
+  # collection, create one and record a timestamp. Otherwise, just increment a total count of enqueued jobs for that group
+  def upsert_group(gid)
+    if job_groups.find_one({'_id' => gid}).nil?
+      job_groups.update( {'_id' => gid},
+      {
+        '$inc' => {'total' => 1},
+        '$set' => {'started_at' => Time.now}
+      }, {:upsert => true})
+    else
+      job_groups.update( {'_id' => gid},
+      {
+        '$inc' => {'total' => 1}
+      }, {:upsert => true})
+    end
+    #Resque.print_groups
+    job_groups.count
+  end
+  
+  # Increments the 'completed' count of the group with id 'gid'.
+  # Also merge the increment operation with optional custom
+  # atomic operations.
+  def record_performed_job(gid, atomic_op = nil)
+    op = {'$inc' => {'completed' => 1}}
+    if atomic_op
+      op['$inc'].merge!(atomic_op.delete('$inc')) if atomic_op['$inc']
+      op.merge!(atomic_op)
+    end
+    job_groups.update( {'_id' => gid}, op , {:upsert => true})
+    #Resque.print_groups
+  end
+  
+  # Increments the 'failed' count of the group with id 'gid'
+  def record_failed_job(gid, exception)
+    job_groups.update( {'_id' => gid},
+    {
+      '$inc' => {'failed' => 1},
+      '$push' => {'exceptions' => exception.inspect}
+    }, {:upsert => true})
+    #Resque.print_groups
+  end
+  
+  def record_delayed_job(gid)
+    job_groups.update( {'_id' => gid},
+    {
+      '$inc' => {'delayed' => 1}
+    }, {:upsert => true})
+    #Resque.print_groups
+  end
+  
+  def delete_delayed_job(gid)
+    job_groups.update( {'_id' => gid},
+    {
+      '$inc' => {'delayed' => -1}
+    }, {:upsert => true})
+    #Resque.print_groups
+  end
+  
+  # Clears all the groups that are being stored in the job_groups collection
+  def reset_groups_queue
+    job_groups.remove
+  end
+  
+  # Deletes a group from the job_groups collection
+  def remove_group(gid)
+    job_groups.remove({'_id' => gid})
+  end
+  
+  # Returns the stats for a given job group
+  def group_stats(gid)
+    job_groups.find_one({'_id' => gid})
+  end
+  
+  def groups_stats(start, count)
+    job_groups.find({}, :skip => start, :limit => count, :sort => ['_id', 1]).to_a
+  end
+  
+  def groups_count
+    job_groups.count
+  end
+  
+  # Given a set of args, looks through all the args that are Hashes for a :group_id key (the inject line replaces string keys with their symbols)
+  def parse_gid(*args)
+    gid = nil
+    args.flatten.each do |arg|
+      next unless arg.kind_of? Hash
+      sym_arg = arg.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+      if !sym_arg[:group_id].nil?
+        gid = sym_arg[:group_id]
+        break
+      end
+    end
+    gid
+  end
+  
+  def print_groups
+    puts "--------------------"
+    job_groups.find().each do |row|
+      puts row.inspect
+    end
+    puts "--------------------"
+  end
+  
+end
+
+Resque.extend Resque::Plugins::Groups
+Resque::Server.class_eval do
+  include ResqueGroups::Server
+end

data/lib/resque_groups/server.rb

@@ -0,0 +1,23 @@
+
+
+# Extend Resque::Server to add tabs
+module ResqueGroups
+  
+  module Server
+
+    def self.included(base)
+
+      base.class_eval do
+
+        get "/groups" do
+          erb File.read(File.join(File.dirname(__FILE__), 'server/views/groups.erb'))
+        end
+      end
+
+    end
+
+    Resque::Server.tabs << 'Groups'
+
+  end
+  
+end

data/lib/resque_groups/server/views/groups.erb

@@ -0,0 +1,43 @@
+<%start = params[:start].to_i %>
+<%count = params[:count] ? params[:count].to_i : 100 %>
+
+<% groups = [resque.groups_stats(start, start + count)].flatten %>
+<% size = resque.groups_count %>
+<h1>Groups</h1>
+
+<p class='intro'>
+  This list below contains the groups being processed.
+</p>
+
+<p class='sub'>
+  Showing <%= start %> to <%= start + groups.size %> of <b><%= size %></b> groups
+</p>
+
+
+<table>
+  <tr>
+    <th>Group</th>
+    <th>Started at</th>
+    <th>Total</th>
+    <th>Delayed</th>
+    <th>Completed</th>
+    <th>Failed</th>
+    <th>Exceptions</th>
+    <th>Custom field</th>
+  </tr>
+  <% groups.each do |group| %>
+    <tr>
+      <td><%= group['_id'] %></td>
+      <td><%= group['started_at'] %></td>
+      <td><%= group['total'] %></td>
+      <td><%= group['delayed'] %></td>
+      <td><%= group['completed'] %></td>
+      <td><%= group['failed'] %></td>
+      <td><%= escape_html(group['exceptions'].inspect) if group['exceptions'] %></td>
+      <td><%= escape_html(group['custom'].inspect) if group['custom'] %></td>
+    </tr>
+  <% end %>
+</table>
+
+
+<%= partial :next_more, :start => start, :count => count, :size => size %>

data/lib/resque_groups/tasks.rb

@@ -0,0 +1,7 @@
+# require 'resque/tasks'
+# will give you the resque tasks
+
+namespace :resque do
+  task :setup
+
+end

data/lib/resque_groups/tracked_job.rb

@@ -0,0 +1,58 @@
+module Resque::Plugins::Groups
+  module TrackedJob
+
+    # The MongoDB atomic operation stored in this variable will be executed
+    # on the group when and if the job is complete.
+    # See http://www.mongodb.org/display/DOCS/Updating for a list of all atomic operations.
+    # e.g.:
+    #   {'$inc' => {'custom.size' => 3}}
+    #   {'$set' => {'custom.api_problem' => true}}
+    # It's highly recommended to use the "custom" field to store your data, even
+    # if no check is done by the plugin.
+    def atomic_op_on_complete
+      @atomic_op_on_complete
+    end
+    def atomic_op_on_complete=(op)
+      @atomic_op_on_complete = op
+    end
+    
+    # resque hook, triggered immediately after a job with *args has been enqueued
+    def after_enqueue_count_job(*args)
+      gid = Resque.parse_gid(*args)
+      raise "The 'group_id' parameter is missing!" if gid.nil?
+      
+      # If this was a scheduled/delayed job, decrement the 'delayed' counter
+      Resque.delete_delayed_job(gid) if !is_delayed(args).nil?
+      
+      # Update the total count for the group in which the enqueued job belongs
+      Resque.upsert_group(gid)
+    end
+    
+    # resque hook, triggered immediately after a job has been performed
+    def after_perform_update_completed(*args)
+      gid = Resque.parse_gid(*args)
+      Resque.record_performed_job(gid, atomic_op_on_complete)
+    end
+    
+    # resque hook, triggered if a job raises any exception. uncaught exceptions propagate up to Resque::Failure
+    def on_failure_update_failed(exception, *args)
+      gid = Resque.parse_gid(*args)
+      Resque.record_failed_job(gid, exception)
+    end
+    
+    # Determines if this was a delayed job or not, based on the :delayed => true arg
+    def is_delayed(*args)
+      delayed = nil
+      args.flatten.each do |arg|
+        next unless arg.kind_of? Hash
+        sym_arg = arg.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+        if !sym_arg[:delayed].nil?
+          delayed = sym_arg[:delayed]
+          break
+        end
+      end
+      delayed
+    end
+    
+  end
+end

data/lib/resque_groups/version.rb

@@ -0,0 +1,7 @@
+require 'resque'
+
+module Resque::Plugins
+  module Groups
+    Version = '0.5'
+  end
+end

data/resque-mongo-groups.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "resque_groups/version"
+
+Gem::Specification.new do |s|
+  s.name        = "resque-mongo-groups"
+  s.version     = Resque::Plugins::Groups::Version 
+  s.platform    = Gem::Platform::RUBY
+  s.author      = 'Gheorghita Catalin Bordianu'
+  s.email       = ['catalin@silentale.com']
+  s.homepage    = "http://github.com/omikronn/resque-mongo-groups"
+  s.summary     = "Keeps track of groups of jobs and their completion status"
+  s.description = %q{Provides a way of keeping track of batches of jobs, completion status, start time etc. using a MongoDb collection}
+  
+  s.required_rubygems_version = ">= 1.3.6"
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  
+  s.rubyforge_project = "resque-mongo-groups"
+
+  s.files        = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
+  s.require_path = 'lib'
+  
+  s.add_runtime_dependency(%q<resque-mongo>, [">= 1.11.0"])
+  
+end

data/tasks/resque_counter.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'resque_groups/tasks'

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: resque-mongo-groups
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: "0.5"
+platform: ruby
+authors: 
+- Gheorghita Catalin Bordianu
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-24 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: resque-mongo
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.11.0
+  type: :runtime
+  version_requirements: *id002
+description: Provides a way of keeping track of batches of jobs, completion status, start time etc. using a MongoDb collection
+email: 
+- catalin@silentale.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .bundle/config
+- .gitignore
+- Gemfile
+- HISTORY.md
+- LICENSE
+- README.markdown
+- Rakefile
+- lib/resque_groups.rb
+- lib/resque_groups/server.rb
+- lib/resque_groups/server/views/groups.erb
+- lib/resque_groups/tasks.rb
+- lib/resque_groups/tracked_job.rb
+- lib/resque_groups/version.rb
+- resque-mongo-groups.gemspec
+- tasks/resque_counter.rake
+has_rdoc: true
+homepage: http://github.com/omikronn/resque-mongo-groups
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: resque-mongo-groups
+rubygems_version: 1.6.1
+signing_key: 
+specification_version: 3
+summary: Keeps track of groups of jobs and their completion status
+test_files: []
+