qless 0.9.3 → 0.10.0

data/Gemfile

@@ -3,8 +3,14 @@ source "http://rubygems.org"
 # Specify your gem's dependencies in qless.gemspec
 gemspec
 
-group :development do
-  gem 'debugger', :platform => :mri
+group :extras do
+  gem 'debugger', :platform => :mri_19
 end
 
-gem 'thin' # needed by qless-web binary
+group :development do
+  gem 'byebug', :platforms => [:ruby_20, :ruby_21]
+  gem 'pry'
+  gem 'pry-byebug', :platforms => [:ruby_20, :ruby_21]
+  gem 'pry-stack_explorer'
+  gem 'cane', :platforms => [:ruby_20, :ruby_21]
+end

data/README.md

@@ -1,4 +1,4 @@
-qless
+qless [![Build Status](https://travis-ci.org/seomoz/qless.svg?branch=master)](https://travis-ci.org/seomoz/qless)
 =====
 
 Qless is a powerful `Redis`-based job queueing system inspired by
@@ -120,7 +120,7 @@ job.original_retries # => the number of times the job is allowed to be retried
 job.retries_left # => the number of retries left
 
 # You can also change the job in various ways:
-job.move("some_other_queue") # move it to a new queue
+job.requeue("some_other_queue") # move it to a new queue
 job.cancel # cancel the job
 job.tag("foo") # add a tag
 job.untag("foo") # remove a tag
@@ -162,31 +162,41 @@ it is empty, before trying to pop job off the second queue. The
 round-robin reserver will pop a job off the first queue, then the second
 queue, and so on. You could also easily implement your own.
 
-To start a worker, load the qless rake tasks in your Rakefile, and
-define a `qless:setup` task:
+To start a worker, write a bit of Ruby code that instantiates a
+worker and runs it. You could write a rake task to do this, for
+example:
 
 ``` ruby
-require 'qless/tasks'
 namespace :qless do
-  task :setup do
-    require 'my_app/environment' # to ensure all job classes are loaded
-
-    # Set options via environment variables
-    # The only required option is QUEUES; the
-    # rest have reasonable defaults.
-    ENV['REDIS_URL'] ||= 'redis://some-host:7000/3'
-    ENV['QUEUES'] ||= 'fizz,buzz'
-    ENV['JOB_RESERVER'] ||= 'Ordered'
-    ENV['INTERVAL'] ||= '10' # 10 seconds
-    ENV['VERBOSE'] ||= 'true'
-  end
-end
-```
+  desc "Run a Qless worker"
+  task :work do
+    # Load your application code. All job classes must be loaded.
+    require 'my_app/environment'
 
-Then run the `qless:work` rake task:
+    # Require the parts of qless you need
+    require 'qless'
+    require 'qless/job_reservers/ordered'
+    require 'qless/worker'
 
-```
-rake qless:work
+    # Create a client
+    client = Qless::Client.new(:host => 'foo.bar.com', :port => 1234)
+
+    # Get the queues you use
+    queues = %w[ queue_1 queue_2 ].map do |name|
+      client.queues[name]
+    end
+
+    # Create a job reserver; different reservers use different
+    # strategies for which order jobs are popped off of queues
+    reserver = Qless::JobReservers::Ordered.new(queues)
+
+    # Create a forking worker that uses the given reserver to pop jobs.
+    worker = Qless::Workers::ForkingWorker.new(reserver)
+
+    # Start the worker!
+    worker.run
+  end
+end
 ```
 
 The following signals are supported in the parent process:
@@ -276,13 +286,13 @@ has a rack-based ruby web app, we recommend you mount Qless's web app
 in it. Here's how you can do that with `Rack::Builder` in your `config.ru`:
 
 ``` ruby
-Qless::Server.client = Qless::Client.new(:host => "some-host", :port => 7000)
+client = Qless::Client.new(:host => "some-host", :port => 7000)
 
 Rack::Builder.new do
   use SomeMiddleware
 
   map('/some-other-app') { run Apps::Something.new }
-  map('/qless')          { run Qless::Server.new }
+  map('/qless')          { run Qless::Server.new(client) }
 end
 ```
 
@@ -506,7 +516,7 @@ progress periodically:
 
 ``` ruby
 # Wait until we have 5 minutes left on the heartbeat, and if we find that
-# we've lost our lock on a job, then honorable fall on our sword
+# we've lost our lock on a job, then honorably fall on our sword
 if (job.ttl < 300) && !job.heartbeat
   return / die / exit
 end
@@ -608,3 +618,38 @@ options a normal job supports. See
 [the source](https://github.com/seomoz/qless/blob/master/lib/qless/job.rb)
 for a full list.
 
+Contributing
+============
+
+To bootstrap an environment, first [have a redis](https://github.com/seomoz/qless/wiki/Bootstrapping-Qless#a-simple-redis-bootstrap).
+
+Have `rvm` or `rbenv`.  Then to install the dependencies:
+
+```bash
+rbenv install                 # rbenv only.  Install bundler if you need it.
+bundle install
+./exe/install_phantomjs       # Bring in phantomjs 1.7.0 for tests.
+rbenv rehash                  # rbenv only
+git submodule init
+git submodule update
+bundle exec rake core:build
+```
+
+To run the tests:
+
+```
+bundle exec rake spec
+```
+
+**The locally installed redis will be flushed before and after each test run.**
+
+To change the redis instance used in tests, put the connection information into [`./spec/redis.config.yml`](https://github.com/seomoz/qless/blob/92904532aee82aaf1078957ccadfa6fcd27ae408/spec/spec_helper.rb#L26).
+
+To contribute, fork the repo, use feature branches, run the tests and open PRs.
+
+Mailing List
+============
+
+For questions and general Qless discussion, please join the [Qless
+Mailing list](https://groups.google.com/forum/?fromgroups#!forum/qless).
+

data/Rakefile

@@ -1,8 +1,6 @@
 #!/usr/bin/env rake
-
-require 'bundler'
-Bundler.setup
-require "bundler/gem_tasks"
+require 'bundler/gem_helper'
+Bundler::GemHelper.install_tasks
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec) do |t|
@@ -11,8 +9,8 @@ RSpec::Core::RakeTask.new(:spec) do |t|
 end
 
 # TODO: bump this up as test coverage increases. It was 90.29 when I last updated it on 2012-05-21.
-# On travis where we skip JS tests, it's at 83.9 on 2013-01-15
-min_coverage_threshold = 83.5
+# On travis where we skip JS tests, it's at 90.0 on 2013-10-01
+min_coverage_threshold = 85.0
 desc "Checks the spec coverage and fails if it is less than #{min_coverage_threshold}%"
 task :check_coverage do
   percent = File.read("./coverage/coverage_percent.txt").to_f
@@ -23,17 +21,85 @@ task :check_coverage do
   end
 end
 
-task default: [:spec, :check_coverage]
+task default: [:spec, :check_coverage, :cane]
+
+namespace :core do
+  qless_core_dir = "./lib/qless/qless-core"
+
+  desc "Builds the qless-core lua scripts"
+  task :build do
+    Dir.chdir(qless_core_dir) do
+      sh "make clean && make"
+      sh "cp qless.lua ../lua"
+      sh "cp qless-lib.lua ../lua"
+    end
+  end
+
+  task :update_submodule do
+    Dir.chdir(qless_core_dir) do
+      sh "git checkout master"
+      sh "git pull --rebase"
+    end
+  end
+
+  desc "Updates qless-core and rebuilds it"
+  task update: [:update_submodule, :build]
+
+  namespace :verify do
+    script_files = %w[ lib/qless/lua/qless.lua lib/qless/lua/qless-lib.lua ]
+
+    desc "Verifies the script has no uncommitted changes"
+    task :clean do
+      script_files.each do |file|
+        git_status = `git status -- #{file}`
+        unless /working directory clean/.match(git_status)
+          raise "#{file} is dirty: \n\n#{git_status}\n\n"
+        end
+      end
+    end
+
+    desc "Verifies the script is current"
+    task :current do
+      require 'digest/md5'
+      our_md5s = script_files.map do |file|
+        Digest::MD5.hexdigest(File.read file)
+      end
+
+      canonical_md5s = Dir.chdir(qless_core_dir) do
+        sh "make clean && make"
+        script_files.map do |file|
+          Digest::MD5.hexdigest(File.read(File.basename file))
+        end
+      end
+
+      unless our_md5s == canonical_md5s
+        raise "The current scripts are out of date with qless-core"
+      end
+    end
+  end
+
+  desc "Verifies the committed script is current"
+  task verify: %w[ verify:clean verify:current ]
+end
+
+desc "Starts a qless console"
+task :console do
+  ENV['PUBLIC_SEQUEL_API'] = 'true'
+  ENV['NO_NEW_RELIC'] = 'true'
+  exec "bundle exec pry -r./conf/console"
+end
 
 require 'qless/tasks'
 
 namespace :qless do
-  task :setup do
+  desc "Runs a test worker so you can send signals to it for testing"
+  task :run_test_worker do
     require 'qless'
+    require 'qless/job_reservers/ordered'
+    require 'qless/worker'
     queue = Qless::Client.new.queues["example"]
     queue.client.redis.flushdb
 
-    ENV['QUEUES'] = queue.name
     ENV['VVERBOSE'] = '1'
 
     class ExampleJob
@@ -48,5 +114,55 @@ namespace :qless do
     20.times do |i|
       queue.put(ExampleJob, sleep: i)
     end
+
+    reserver = Qless::JobReservers::Ordered.new([queue])
+    Qless::Workers::ForkingWorker.new(reserver, log_level: Logger::INFO).run
+  end
+end
+
+
+namespace :cane do
+  begin
+    require 'cane/rake_task'
+
+    libs = [
+      { name: 'qless', dir: '.', root: '.' },
+    ]
+
+    libs.each do |lib|
+      desc "Runs cane code quality checks for #{lib[:name]}"
+      Cane::RakeTask.new(lib[:name]) do |cane|
+        cane.no_doc   = true
+
+        cane.abc_glob = "#{lib[:dir]}/{lib,spec}/**/*.rb"
+        cane.abc_max = 15
+        cane.abc_exclude = %w[
+          Middleware::(anon)#expect_job_to_timeout
+          Qless::Job#initialize
+          Qless::Middleware::RequeueExceptions#handle_exception
+          Qless::Middleware::Timeout#initialize
+          Qless::WorkerHelpers#run_jobs
+          Qless::Workers::BaseWorker#initialize
+          Qless::Workers::BaseWorker#register_signal_handlers
+          Qless::Workers::ForkingWorker#register_signal_handlers
+          Qless::Workers::SerialWorker#run
+        ]
+
+        cane.style_glob = "#{lib[:dir]}/lib/**/*.rb"
+        cane.style_measure = 100
+        cane.style_exclude = %w[
+        ]
+      end
+    end
+
+    desc "Runs cane code quality checks for all projects"
+    task all: libs.map { |l| l[:name] }
+
+  rescue LoadError
+    task :all do
+      puts "cane is not supported in ruby #{RUBY_VERSION}"
+    end
   end
 end
+
+task cane: "cane:all"

data/exe/install_phantomjs

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+os_name=$(uname -s)
+phantomjs_version=1.7.0
+
+if [[ "$os_name" == 'Darwin' ]]
+then
+  # brew versions phantomjs
+  (cd /usr/local/Library && git checkout d37d922 Formula/phantomjs.rb)
+  brew install phantomjs
+elif [[ "$os_name" == 'Linux' ]]
+then
+  version=phantomjs-1.7.0-linux-x86_64
+  wget http://phantomjs.googlecode.com/files/$version.tar.bz2
+  tar xjf $version.tar.bz2
+  mv $version phantomjs
+else
+  echo "Don't know how to install phantom js on $os_name.  " \
+    "Your OS is bad! You should feel bad!" >&2
+  exit 1
+fi

data/lib/qless.rb

@@ -1,28 +1,34 @@
-require "socket"
-require "redis"
-require "json"
-require "securerandom"
+# Encoding: utf-8
 
+require 'socket'
+require 'redis'
+require 'json'
+require 'securerandom'
+
+# The top level container for all things qless
 module Qless
-  # Define our error base class before requiring the other
-  # files so they can define subclasses.
+  # Define our error base class before requiring the other files so they can
+  # define subclasses.
   Error = Class.new(StandardError)
 
-  # to maintain backwards compatibility with v2.x of that gem we need this constant because:
-  # * (lua.rb) the #evalsha method signature changed between v2.x and v3.x of the redis ruby gem
-  # * (worker.rb) in v3.x you have to reconnect to the redis server after forking the process
+  # to maintain backwards compatibility with v2.x of that gem we need this
+  # constant because:
+  # - (lua.rb) the #evalsha method signature changed between v2.x and v3.x of
+  #   the redis ruby gem
+  # - (worker.rb) in v3.x you have to reconnect to the redis server after
+  #   forking the process
   USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
 end
 
-require "qless/version"
-require "qless/config"
-require "qless/queue"
-require "qless/job"
-require "qless/lua_script"
+require 'qless/version'
+require 'qless/config'
+require 'qless/queue'
+require 'qless/job'
+require 'qless/lua_script'
+require 'qless/failure_formatter'
 
+# The top level container for all things qless
 module Qless
-  extend self
-
   UnsupportedRedisVersionError = Class.new(Error)
 
   def generate_jid
@@ -35,74 +41,93 @@ module Qless
     end
   end
 
-  # This is a unique identifier for the worker
-  def worker_name
-    @worker_name ||= [Socket.gethostname, Process.pid.to_s].join('-')
+  def failure_formatter
+    @failure_formatter ||= FailureFormatter.new
   end
 
+  module_function :generate_jid, :stringify_hash_keys, :failure_formatter
+
+  # A class for interacting with jobs. Not meant to be instantiated directly,
+  # it's accessed through Client#jobs
   class ClientJobs
     def initialize(client)
       @client = client
     end
 
-    def complete(offset=0, count=25)
-      @client._jobs.call([], ['complete', offset, count])
+    def complete(offset = 0, count = 25)
+      @client.call('jobs', 'complete', offset, count)
     end
 
     def tracked
-      results = JSON.parse(@client._track.call([], []))
+      results = JSON.parse(@client.call('track'))
       results['jobs'] = results['jobs'].map { |j| Job.new(@client, j) }
       results
     end
 
-    def tagged(tag, offset=0, count=25)
-      JSON.parse(@client._tag.call([], ['get', tag, offset, count]))
+    def tagged(tag, offset = 0, count = 25)
+      results = JSON.parse(@client.call('tag', 'get', tag, offset, count))
+      # Should be an empty array instead of an empty hash
+      results['jobs'] = [] if results['jobs'] == {}
+      results
     end
 
-    def failed(t=nil, start=0, limit=25)
-      if not t
-        JSON.parse(@client._failed.call([], []))
+    def failed(t = nil, start = 0, limit = 25)
+      if !t
+        return JSON.parse(@client.call('failed'))
       else
-        results = JSON.parse(@client._failed.call([], [t, start, limit]))
-        results['jobs'] = results['jobs'].map { |j| Job.new(@client, j) }
+        results = JSON.parse(@client.call('failed', t, start, limit))
+        results['jobs'] = multiget(*results['jobs'])
         results
       end
     end
 
     def [](id)
-      results = @client._get.call([], [id])
+      get(id)
+    end
+
+    def get(jid)
+      results = @client.call('get', jid)
       if results.nil?
-        results = @client._recur.call([], ['get', id])
-        if results.nil?
-          return nil
-        end
+        results = @client.call('recur.get', jid)
+        return nil if results.nil?
         return RecurringJob.new(@client, JSON.parse(results))
       end
       Job.new(@client, JSON.parse(results))
     end
+
+    def multiget(*jids)
+      results = JSON.parse(@client.call('multiget', *jids))
+      results.map do |data|
+        Job.new(@client, data)
+      end
+    end
   end
 
+  # A class for interacting with workers. Not meant to be instantiated
+  # directly, it's accessed through Client#workers
   class ClientWorkers
     def initialize(client)
       @client = client
     end
 
     def counts
-      JSON.parse(@client._workers.call([], [Time.now.to_i]))
+      JSON.parse(@client.call('workers'))
     end
 
     def [](name)
-      JSON.parse(@client._workers.call([], [Time.now.to_i, name]))
+      JSON.parse(@client.call('workers', name))
     end
   end
 
+  # A class for interacting with queues. Not meant to be instantiated directly,
+  # it's accessed through Client#queues
   class ClientQueues
     def initialize(client)
       @client = client
     end
 
     def counts
-      JSON.parse(@client._queues.call([], [Time.now.to_i]))
+      JSON.parse(@client.call('queues'))
     end
 
     def [](name)
@@ -110,29 +135,28 @@ module Qless
     end
   end
 
+  # A class for interacting with events. Not meant to be instantiated directly,
+  # it's accessed through Client#events
   class ClientEvents
+    EVENTS = %w{canceled completed failed popped stalled put track untrack}
+    EVENTS.each do |method|
+      define_method(method.to_sym) do |&block|
+        @actions[method.to_sym] = block
+      end
+    end
+
     def initialize(redis)
       @redis   = redis
-      @actions = Hash.new()
+      @actions = {}
     end
 
-    def canceled(&block) ; @actions[:canceled ] = block; end
-    def completed(&block); @actions[:completed] = block; end
-    def failed(&block)   ; @actions[:failed   ] = block; end
-    def popped(&block)   ; @actions[:popped   ] = block; end
-    def stalled(&block)  ; @actions[:stalled  ] = block; end
-    def put(&block)      ; @actions[:put      ] = block; end
-    def track(&block)    ; @actions[:track    ] = block; end
-    def untrack(&block)  ; @actions[:untrack  ] = block; end
-
     def listen
       yield(self) if block_given?
-      @redis.subscribe(:canceled, :completed, :failed, :popped, :stalled, :put, :track, :untrack) do |on|
+      channels = EVENTS.map { |event| "ql:#{event}" }
+      @redis.subscribe(channels) do |on|
         on.message do |channel, message|
-          callback = @actions[channel.to_sym]
-          if not callback.nil?
-            callback.call(message)
-          end
+          callback = @actions[channel.sub('ql:', '').to_sym]
+          callback.call(message) unless callback.nil?
         end
       end
     end
@@ -142,29 +166,25 @@ module Qless
     end
   end
 
+  # The client for interacting with Qless
   class Client
-    # Lua scripts
-    attr_reader :_cancel, :_config, :_complete, :_fail, :_failed, :_get, :_heartbeat, :_jobs, :_peek, :_pop
-    attr_reader :_priority, :_put, :_queues, :_recur, :_retry, :_stats, :_tag, :_track, :_workers, :_depends
-    attr_reader :_pause, :_unpause, :_deregister_workers
-    # A real object
-    attr_reader :config, :redis, :jobs, :queues, :workers
+    # Lua script
+    attr_reader :_qless, :config, :redis, :jobs, :queues, :workers
+    attr_accessor :worker_name
 
     def initialize(options = {})
-      # This is the redis instance we're connected to
-      @redis   = options[:redis] || Redis.connect(options) # use connect so REDIS_URL will be honored
+      # This is the redis instance we're connected to. Use connect so REDIS_URL
+      # will be honored
+      @redis   = options[:redis] || Redis.connect(options)
       @options = options
-      assert_minimum_redis_version("2.5.5")
+      assert_minimum_redis_version('2.5.5')
       @config = Config.new(self)
-      ['cancel', 'config', 'complete', 'depends', 'fail', 'failed', 'get', 'heartbeat', 'jobs', 'peek', 'pop',
-        'priority', 'put', 'queues', 'recur', 'retry', 'stats', 'tag', 'track', 'workers', 'pause', 'unpause',
-        'deregister_workers'].each do |f|
-        self.instance_variable_set("@_#{f}", Qless::LuaScript.new(f, @redis))
-      end
+      @_qless = Qless::LuaScript.new('qless', @redis)
 
       @jobs    = ClientJobs.new(self)
       @queues  = ClientQueues.new(self)
       @workers = ClientWorkers.new(self)
+      @worker_name = [Socket.gethostname, Process.pid.to_s].join('-')
     end
 
     def inspect
@@ -178,39 +198,58 @@ module Qless
       @events ||= ClientEvents.new(Redis.connect(@options))
     end
 
+    def call(command, *argv)
+      @_qless.call(command, Time.now.to_f, *argv)
+    end
+
     def track(jid)
-      @_track.call([], ['track', jid, Time.now.to_i])
+      call('track', 'track', jid)
     end
 
     def untrack(jid)
-      @_track.call([], ['untrack', jid, Time.now.to_i])
+      call('track', 'untrack', jid)
     end
 
-    def tags(offset=0, count=100)
-      JSON.parse(@_tag.call([], ['top', offset, count]))
+    def tags(offset = 0, count = 100)
+      JSON.parse(call('tag', 'top', offset, count))
     end
 
     def deregister_workers(*worker_names)
-      _deregister_workers.call([], worker_names)
+      call('worker.deregister', *worker_names)
     end
 
     def bulk_cancel(jids)
-      @_cancel.call([], jids)
+      call('cancel', jids)
+    end
+
+    if ::Redis.instance_method(:dup).owner == ::Redis
+      def new_redis_connection
+        redis.dup
+      end
+    else # redis version < 3.0.7
+      def new_redis_connection
+        ::Redis.new(@options)
+      end
+    end
+
+    def ==(other)
+      self.class == other.class && redis.id == other.redis.id
     end
+    alias eql? ==
 
-    def new_redis_connection
-      ::Redis.new(url: redis.id)
+    def hash
+      self.class.hash ^ redis.id.hash
     end
 
   private
 
     def assert_minimum_redis_version(version)
       # remove the "-pre2" from "2.6.8-pre2"
-      redis_version = @redis.info.fetch("redis_version").split('-').first
+      redis_version = @redis.info.fetch('redis_version').split('-').first
       return if Gem::Version.new(redis_version) >= Gem::Version.new(version)
 
       raise UnsupportedRedisVersionError,
-        "You are running redis #{redis_version}, but qless requires at least #{version}"
+            "Qless requires #{version} or better, not #{redis_version}"
     end
   end
 end