cyclop 0.1.0

data/lib/cyclop.rb

@@ -0,0 +1,131 @@
+require "mongo"
+require "logger"
+require "socket"
+
+require "cyclop/job"
+require "cyclop/worker"
+require "cyclop/action"
+require "cyclop/version"
+
+module Cyclop
+  extend self
+
+  # Raised if db not set or connection error
+  class DatabaseNotAvailable < StandardError; end
+  # Raised if two actions share the same queue
+  class ActionQueueClash < StandardError; end
+  # Raised if no action has been found
+  class NoActionFound < StandardError; end
+
+  # Set which `Mongo::DB` to use
+  def db=(db)
+    @db = db
+  end
+
+  # Get `Mongo::DB` to use
+  def db
+    @db
+  end
+
+  # Get memoized host
+  def host
+    @host ||= Socket.gethostname
+  end
+
+  # Get a unique identifier for current process
+  def master_id
+    @master_id ||= "#{host}-#{Process.pid}-#{Thread.current}"
+  end
+
+  # Queues a new job
+  #
+  #   Minimum usage:
+  #
+  #     Cyclop.push queue: "refresh_cache"
+  #
+  #   With `:job_params`:
+  #
+  #     # with an `Array`
+  #     Cyclop.push queue: "email", job_params: ["1", :welcome]
+  #
+  #     # with a `Hash`
+  #     Cyclop.push queue: "email", job_params: {user_id: "1",
+  #     type: "welcome"}
+  #
+  #   With `:delay`:
+  #
+  #     # Will not perform the task before a delay of 60 seconds
+  #     Cyclop.push queue: "email", delay: 60
+  #
+  #   With `:retries` and `:splay`:
+  #
+  #     # Will mark the task as failed only after 3 retries
+  #     Cyclop.push queue: "email", retries: 3
+  #
+  #     # Will mark the task as failed only after 2 retries
+  #     # and 30 seconds between each retry
+  #     Cyclop.push queue: "email", retries: 2, splay: 30
+  #
+  # Parameters:
+  #
+  #   * (Hash) opts (defaults to: {}) - a customizable set of options. The minimum required is :queue.
+  #
+  # Options Hash (opts):
+  #
+  #   * (Symbol, String) :queue - name of the queue (required).
+  #   * (Array, Hash) :job_params (nil) - parameters to send to the `Cyclop::Job#perform` method.
+  #   * (Integer) :delay (0) - time to wait in `seconds` before the task should be performed.
+  #   * (Integer) :retries (0) - number of retries before the `Cyclop::Job` is marked as failed.
+  #   * (Integer) :splay (60) - time to wait in `seconds` between retry.
+  #   * (String) :host (Cyclop.host) - host under which the `Cyclop::Job` should be added.
+  #
+  # Returns a `Cyclop::Job`
+  #
+  def push(opts={})
+    Cyclop::Job.create opts
+  end
+
+  # Get a `Cyclop::Job` to process
+  #
+  # Parameters:
+  #
+  #   * (Symbol, String) queues - list of queues to get a `Cyclop::Job` from. Defaults to all.
+  #   * (Hash) opts (defaults to: {}) - a customizable set of options.
+  #
+  # Options Hash (opts):
+  #
+  #   * (String) :host - limit to `Cyclop::Job`s queued by this host.
+  #
+  # Returns a `Cyclop::Job` or `nil` if nothing to process
+  #
+  def next(*args)
+    opts = extract_opts! args
+    Cyclop::Job.next({queues: args, locked_by: master_id}.merge opts)
+  end
+
+  # Get failed `Cyclop::Job`s
+  #
+  # Parameters:
+  #
+  #   * (Symbol, String) queues - list of queues to get a `Cyclop::Job` from. Defaults to all.
+  #   * (Hash) opts (defaults to: {}) - a customizable set of options.
+  #
+  # Options Hash (opts):
+  #
+  #   * (String) :host - limit to `Cyclop::Job`s queued by this host.
+  #   * (Integer) :skip (0) - number of `Cyclop::Job`s to skip.
+  #   * (Integer) :limit (nil) - maximum number of `Cyclop::Job`s to return.
+  #
+  # Returns an `Array` of failed `Cyclop::Job`
+  #
+  def failed(*args)
+    opts = extract_opts! args
+    Cyclop::Job.failed({queues: args}.merge opts)
+  end
+
+private
+
+  def extract_opts!(args)
+    (args.pop if args.last.is_a?(Hash)) || {}
+  end
+end

data/spec/cyclop/action_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+describe Cyclop::Action do
+  describe ".find_by_queue(queue)" do
+    context "with no subclass" do
+      it "raises an Cyclop::NoActionFound" do
+        lambda {
+          Cyclop::Action.find_by_queue("email")
+        }.should raise_error Cyclop::NoActionFound, "No action defined"
+      end
+    end
+    context "with subclasses" do
+      before :all do
+        class EmailAction < Cyclop::Action
+          def self.queues
+            ["email", "email_welcome"]
+          end
+        end
+        class CacheAction < Cyclop::Action
+          def self.queues
+            ["cache"]
+          end
+        end
+      end
+      it "returns only action matching queue" do
+        Cyclop::Action.find_by_queue("email").should be EmailAction
+      end
+      it "returns nil if no action match queue" do
+        lambda {
+          Cyclop::Action.find_by_queue("compress")
+        }.should raise_error Cyclop::NoActionFound, 'No action found for "compress" queue. Valid queues: "email", "email_welcome", "cache"'
+      end
+      it "raises an error if two actions share the same queue" do
+        class EmailCacheAction < Cyclop::Action
+          def self.queues
+            ["email", "cache"]
+          end
+        end
+        lambda {
+          Cyclop::Action.find_by_queue "email"
+        }.should raise_error Cyclop::ActionQueueClash, '"email" queue belongs to multiple actions: EmailAction, EmailCacheAction'
+        Object.send :remove_const, :EmailCacheAction
+      end
+    end
+  end
+end

data/spec/cyclop/job_spec.rb

@@ -0,0 +1,273 @@
+require "spec_helper"
+
+describe Cyclop::Job do
+  subject { Cyclop::Job.new queue: "demo" }
+
+  its(:queue){ should == "demo" }
+  its(:job_params){ should be_nil }
+  its(:delay){ should == 0 }
+  its(:delayed_until){ should be_nil }
+  its(:retries){ should == 0 }
+  its(:splay){ should == 60 }
+  its(:created_by){ should == Cyclop.host }
+  its(:created_at){ should be_nil }
+  its(:updated_at){ should be_nil }
+  its(:locked_by){ should be_nil }
+  its(:locked_at){ should be_nil }
+  its(:failed){ should be false }
+  its(:attempts){ should == 0 }
+  its(:errors){ should == [] }
+
+  describe "#save" do
+    it "raises a Cyclop::DatabaseNotAvailable if no db defined" do
+      old, Cyclop.db = Cyclop.db, nil
+      lambda {
+        Cyclop::Job.new(queue: "demo").save
+      }.should raise_error Cyclop::DatabaseNotAvailable
+      Cyclop.db = old
+    end
+    context "on create" do
+      it "saves the job" do
+        subject.save.should be_true
+        subject.should be_persisted
+        subject.reload.queue.should == "demo"
+      end
+    end
+  end
+
+  describe ".create(opts={})" do
+    it do
+      lambda {
+        Cyclop::Job.create
+      }.should raise_error ArgumentError, ":queue is required"
+    end
+    it "returns a persisted job on success" do
+      job = Cyclop::Job.create queue: "demo"
+      job.should be_persisted
+    end
+  end
+
+  describe ".next(opts={})" do
+    it do
+      lambda {
+        Cyclop::Job.next
+      }.should raise_error ArgumentError, "locked_by is required"
+    end
+    context "with no job in queue" do
+      it "returns nil" do
+        Cyclop::Job.next(locked_by: "myid").should be_nil
+      end
+    end
+    context "with jobs in queue" do
+      let(:email_job_delayed) { Cyclop::Job.create queue: "email", delay: 3600 }
+      let(:email_job_failed) { Cyclop::Job.create queue: "email", failed: true }
+      let(:email_job_no_retry) { Cyclop::Job.create queue: "email", attempts: 1 }
+      let(:email_job_locked) { Cyclop::Job.create queue: "email", locked_at: Time.now.utc-10 }
+      let(:email_job_next) { Cyclop::Job.create queue: "email" }
+      let(:email_job_other_host) { Cyclop::Job.create queue: "email", created_by: "demo.local" }
+      let(:snail_mail_job_stalling) { Cyclop::Job.create queue: "snail mail", locked_at: Time.now.utc-3000 }
+
+      before do
+        email_job_delayed
+        email_job_failed
+        email_job_no_retry
+        email_job_next
+        email_job_other_host
+        snail_mail_job_stalling
+      end
+
+      it "returns the first job to process when called without queue" do
+        Cyclop::Job.next(locked_by: "myid").should == email_job_next
+      end
+
+      it "increments attempts" do
+        Cyclop::Job.next(locked_by: "myid").attempts.should == 1
+      end
+
+      it "locks to given id" do
+        Cyclop::Job.next(locked_by: "myid").locked_by.should == "myid"
+      end
+
+      it "sets locked_at" do
+        Cyclop::Job.next(locked_by: "myid").locked_at.should_not be_nil
+      end
+
+      it "returns the first job to process when called with :email queue" do
+        Cyclop::Job.next(queues: [:email], locked_by: "myid").should == email_job_next
+      end
+
+      it "returns the first job to process for a given host" do
+        Cyclop::Job.next(queues: [:email], locked_by: "myid", host: "demo.local").should == email_job_other_host
+      end
+
+      it "returns nil when called with :cache queue" do
+        Cyclop::Job.next(queues: [:cache], locked_by: "myid").should be_nil
+      end
+
+      it "unlock and returns job stalled for a long time" do
+        Cyclop::Job.next(queues: ["snail mail"], locked_by: "myid").should == snail_mail_job_stalling
+      end
+    end
+  end
+  describe ".failed(opts={})" do
+    context "with no job in queue" do
+      it "returns an empty array" do
+        Cyclop::Job.failed.should == []
+      end
+    end
+    context "with jobs in queue" do
+      let(:email_job_failed) { Cyclop::Job.create queue: "email", failed: true }
+      let(:cache_job_no_retry) { Cyclop::Job.create queue: "cache", attempts: 1 }
+      let(:cache_job_failed) { Cyclop::Job.create queue: "cache", failed: true }
+      let(:email_job_next) { Cyclop::Job.create queue: "email" }
+
+      before do
+        email_job_failed
+        cache_job_no_retry
+        cache_job_failed
+        email_job_next
+      end
+
+      it "returns all failed jobs when called without queue" do
+        Cyclop::Job.failed.should == [email_job_failed, cache_job_no_retry, cache_job_failed]
+      end
+
+      it "returns all failed job to process when called with :email queue" do
+        Cyclop::Job.failed(queues: [:email]).should == [email_job_failed]
+      end
+
+      it "returns empty array when called with :snail_mail queue" do
+        Cyclop::Job.failed(queues: [:snail_mail]).should == []
+      end
+
+      it "respect :skip and :limit options" do
+        Cyclop::Job.failed(skip: 1, limit: 1).should == [cache_job_no_retry]
+      end
+    end
+  end
+  describe "#complete!" do
+    context "when locked by the same process" do
+      let(:email_job) { Cyclop::Job.create queue: "email", locked_by: Cyclop.master_id, locked_at: Time.now.utc }
+      it "removes the job" do
+        email_job.complete!
+        Cyclop::Job.find(email_job._id).should be_nil
+      end
+    end
+    context "when locked by another process" do
+      let(:email_job) { Cyclop::Job.create queue: "email", locked_by: "anotherid", locked_at: Time.now.utc }
+      it "keeps the job" do
+        email_job.complete!
+        Cyclop::Job.find(email_job._id).should == email_job
+      end
+    end
+  end
+  describe "#release!" do
+    context "without exception" do
+      context "when locked by the same process and no more retries to do" do
+        let(:email_job) { Cyclop::Job.create queue: "email", locked_by: Cyclop.master_id, locked_at: ::Time.at(Time.now.to_i).utc, attempts: 1 }
+        before :all do
+          email_job.release!
+          @reload = Cyclop::Job.find email_job._id
+        end
+        it "marks it as failed" do
+          @reload.failed.should be_true
+        end
+        it "keeps locked_at" do
+          @reload.locked_at.should == email_job.locked_at
+        end
+        it "keeps locked_by" do
+          @reload.locked_by.should == email_job.locked_by
+        end
+      end
+      context "when locked by the same process and more retries to do" do
+        let(:email_job) { Cyclop::Job.create queue: "email", locked_by: Cyclop.master_id, locked_at: ::Time.at(Time.now.to_i).utc, attempts: 1, retries: 1, splay: 1 }
+        before :all do
+          email_job.release!
+          @reload = Cyclop::Job.find email_job._id
+        end
+        it "marks it as failed" do
+          @reload.failed.should be_false
+        end
+        it "clears locked_at" do
+          @reload.locked_at.should be_nil
+        end
+        it "clears locked_by" do
+          @reload.locked_by.should be_nil
+        end
+        it "sets delayed_until based on splay" do
+          @reload.delayed_until.should == email_job.locked_at+1
+        end
+      end
+      context "when locked by another process and no more retries to do" do
+        let(:email_job) { Cyclop::Job.create queue: "email", locked_by: "anotherid", locked_at: Time.now.utc, attempts: 1 }
+        it "doesn't mark it as failed" do
+          email_job.release!
+          Cyclop::Job.find(email_job._id).failed.should be_false
+        end
+      end
+    end
+    context "with exception" do
+      let(:exception) { mock :class => Exception, :message => "Soft fail", :backtrace => "backtrace" }
+      before do
+        email_job.release! exception
+        @reload = Cyclop::Job.find email_job._id
+      end
+      context "when locked by the same process and no more retries to do" do
+        let(:email_job) { Cyclop::Job.create queue: "email", locked_by: Cyclop.master_id, locked_at: ::Time.at(Time.now.to_i).utc, attempts: 1 }
+        it "marks it as failed" do
+          @reload.failed.should be_true
+        end
+        it "keeps locked_at" do
+          @reload.locked_at.should == email_job.locked_at
+        end
+        it "keeps locked_by" do
+          @reload.locked_by.should == email_job.locked_by
+        end
+        it "has recorded the error" do
+          @reload.errors.should have(1).item
+          error = @reload.errors.first
+          error["locked_by"].should == email_job.locked_by
+          error["locked_at"].should == email_job.locked_at
+          error["class"].should == "Exception"
+          error["message"].should == "Soft fail"
+          error["backtrace"].should == "backtrace"
+          error["created_at"].should_not be_nil
+        end
+      end
+      context "when locked by the same process and more retries to do" do
+        let(:email_job) { Cyclop::Job.create queue: "email", locked_by: Cyclop.master_id, locked_at: ::Time.at(Time.now.to_i).utc, attempts: 1, retries: 2, splay: 1 }
+        it "marks it as failed" do
+          @reload.failed.should be_false
+        end
+        it "clears locked_at" do
+          @reload.locked_at.should be_nil
+        end
+        it "clears locked_by" do
+          @reload.locked_by.should be_nil
+        end
+        it "sets delayed_until based on splay" do
+          @reload.delayed_until.should == email_job.locked_at+1
+        end
+        it "has recorded the error" do
+          @reload.errors.should have(1).item
+          error = @reload.errors.first
+          error["locked_by"].should == email_job.locked_by
+          error["locked_at"].should == email_job.locked_at
+          error["class"].should == "Exception"
+          error["message"].should == "Soft fail"
+          error["backtrace"].should == "backtrace"
+          error["created_at"].should_not be_nil
+        end
+      end
+      context "when locked by another process and no more retries to do" do
+        let(:email_job) { Cyclop::Job.create queue: "email", locked_by: "anotherid", locked_at: Time.now.utc, attempts: 1 }
+        it "doesn't mark it as failed" do
+          @reload.failed.should be_false
+        end
+        it "doesn't add error" do
+          @reload.errors.should be_empty
+        end
+      end
+    end
+  end
+end

data/spec/cyclop/worker_spec.rb

@@ -0,0 +1,78 @@
+require "spec_helper"
+
+describe Cyclop::Worker do
+  subject { Cyclop::Worker.new({"mongo" => {"database" => "cyclop_test"}}) }
+
+  its(:queues){ should be_empty }
+  its(:logger){ should_not be_nil }
+  its(:sleep_interval){ should == 1 }
+  its(:actions){ should == "./actions" }
+
+  it "raise ArgumentError without mongo['database']" do
+    lambda {
+      Cyclop::Worker.new
+    }.should raise_error ArgumentError, 'mongo["database"] is required'
+  end
+
+  describe "#perform" do
+    let(:worker) do
+      Cyclop::Worker.new({
+        "mongo" => {"database" => "cyclop_test"},
+        "actions" => File.expand_path("../../fixtures/actions", __FILE__),
+      }) 
+    end
+    let(:job) { Cyclop.push queue: "slow", job_params: ["tony@starkenterprises.com", :welcome] }
+    it "loads actions files" do
+      lambda { Cyclop::Spec::Action::Email }.should raise_error NameError
+      worker.perform job
+      lambda { Cyclop::Spec::Action::Email }.should_not raise_error
+    end
+    it "calls perform on action class with specified params" do
+      require File.expand_path("../../fixtures/actions/email", __FILE__)
+      Cyclop::Spec::Action::Email.should_receive(:perform).with("tony@starkenterprises.com", :welcome)
+      worker.perform job
+    end
+  end
+  
+  describe "#run" do
+    let(:worker) do
+      Cyclop::Worker.new({
+        "log_file" => File.expand_path("../../../test.log", __FILE__),
+        "mongo" => {"database" => "cyclop_test"},
+        "actions" => File.expand_path("../../fixtures/actions", __FILE__),
+      }) 
+    end
+    context "with successful action" do
+      it "remove the job" do
+        job = Cyclop.push queue: "slow", job_params: ["tony@starkenterprises.com", :welcome]
+        t = Thread.new { worker.run }
+        sleep 1
+        worker.stop
+        t.join
+        Cyclop::Job.find(job._id).should be_nil
+      end
+    end
+
+    context "with failing action" do
+      it "mark the job as failed" do
+        job = Cyclop.push queue: "slow", job_params: ["tony@starkenterprises.com"]
+        t = Thread.new { worker.run }
+        sleep 1
+        worker.stop
+        t.join
+        job.reload.failed.should be_true
+      end
+
+      it "mark the job as failed after retry" do
+        job = Cyclop.push queue: "slow", job_params: ["tony@starkenterprises.com"], retries: 1, splay: 0
+        t = Thread.new { worker.run }
+        sleep 1
+        worker.stop
+        t.join
+        job.reload
+        job.failed.should be_true
+        job.attempts.should == 2
+      end
+    end
+  end
+end

data/spec/cyclop_spec.rb

@@ -0,0 +1,39 @@
+require "spec_helper"
+
+describe Cyclop do
+  describe ".push(opts={})" do
+    it "creates a new Cyclop::Job initialized with opts" do
+      opts = {}
+      Cyclop::Job.should_receive :create, with: opts
+      Cyclop.push opts
+    end
+  end
+  describe ".next(*args)" do
+    it "extracts queues from args" do
+      Cyclop::Job.should_receive(:next).with(queues: [:email, "cache"], locked_by: Cyclop.send(:master_id))
+      Cyclop.next :email, "cache"
+    end
+    it "extracts options from args" do
+      Cyclop::Job.should_receive(:next).with(queues: [:email, "cache"], locked_by: "myid")
+      Cyclop.next :email, "cache", locked_by: "myid"
+    end
+    it "extracts options from args even without queues" do
+      Cyclop::Job.should_receive(:next).with(queues: [], locked_by: "myid")
+      Cyclop.next locked_by: "myid"
+    end
+  end
+  describe ".failed(opts={})" do
+    it "extracts queues from args" do
+      Cyclop::Job.should_receive(:failed).with(queues: [:email, "cache"])
+      Cyclop.failed :email, "cache"
+    end
+    it "extracts options from args" do
+      Cyclop::Job.should_receive(:failed).with(queues: [:email, "cache"], limit: 10)
+      Cyclop.failed :email, "cache", limit: 10
+    end
+    it "extracts options from args even without queues" do
+      Cyclop::Job.should_receive(:failed).with(queues: [], limit: 10)
+      Cyclop.failed limit: 10
+    end
+  end
+end

data/spec/fixtures/actions/email.rb

@@ -0,0 +1,14 @@
+module Cyclop
+  module Spec
+    module Action
+      class Email < Cyclop::Action
+        def self.queues
+          ["slow"]
+        end
+
+        def self.perform(to, kind)
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require "bundler/setup"
+require "logger"
+require "cyclop"
+
+Dir[File.expand_path("../support/**/*.rb", __FILE__)].each{|f| require f}
+
+RSpec.configure do |config|
+  config.before(:all) do
+    logger = Logger.new "test.log"
+    Cyclop.db = Mongo::Connection.new("localhost", nil, :logger => logger).db "cyclop_test"
+  end
+  config.before do
+    Cyclop.db.collections.each(&:remove)
+  end
+end