RubyGems - cyclop - Versions diffs - 0.1.0 - Mend

cyclop 0.1.0

Files changed (19) hide show

data/.gitignore +7 -0
data/.rspec +2 -0
data/Gemfile +3 -0
data/README.md +88 -0
data/Rakefile +59 -0
data/bin/cyclop +81 -0
data/cyclop.gemspec +29 -0
data/lib/cyclop/action.rb +34 -0
data/lib/cyclop/job.rb +211 -0
data/lib/cyclop/version.rb +3 -0
data/lib/cyclop/worker.rb +128 -0
data/lib/cyclop.rb +131 -0
data/spec/cyclop/action_spec.rb +46 -0
data/spec/cyclop/job_spec.rb +273 -0
data/spec/cyclop/worker_spec.rb +78 -0
data/spec/cyclop_spec.rb +39 -0
data/spec/fixtures/actions/email.rb +14 -0
data/spec/spec_helper.rb +15 -0
metadata +141 -0

data/.gitignore ADDED Viewed

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+docs/
+*.log
+*.yml

data/.rspec ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ --color
2	+ --format documentation

data/Gemfile ADDED Viewed

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+gemspec

data/README.md ADDED Viewed

@@ -0,0 +1,88 @@
+Cyclop
+======
+Job queue with MongoDB with emphasis on never losing any task even if worker fails hard (segfault).
+Dependencies
+------------
+* Ruby >= 1.9.2
+* gem "mongo", "~> 1.3.1"
+* gem "posix-spawn", "~> 0.3.6"
+Usage
+-----
+* Give Cyclop access to mongo:
+        Cyclop.db = Mongo::Connection.new["database_name"]
+    or with Replica-Sets
+        Cyclop.db = Mongo::ReplSetConnection.new["database_name"]
+    or if you're using MongoMapper:
+        Cyclop.db = MongoMapper.database
+    or if you're using Mongoid:
+        Cyclop.db = Mongoid.database
+* Queue a new task:
+        Cyclop.push({
+          queue: :upload,
+          job_params: {
+            url: "http://example.com",
+          },
+        })
+* Queue a new task to process in 5 minutes, to retry 3 times in case of error with a 1 minute delay between each:
+        Cyclop.push({
+          queue: :convert,
+          job_params: {
+            tmp_file: "/tmp/uploaded_file_32.png",
+          },
+          delay: 300,
+          retries: 3,
+          splay: 60,
+        })
+* Get next job:
+        Cyclop.next
+* Get next job on specific queues:
+        Cyclop.next :upload, :convert
+* Get next job on specific queues for a specific host:
+        Cyclop.next :upload, :convert, host: "tartarus.local"
+* Get failed jobs (limit to 30):
+        Cyclop.failed limit: 30
+* Get failed jobs (skip first 10, limit to 30):
+        Cyclop.failed skip: 10, limit: 30
+* Requeue a failed job:
+        job = Cyclop.failed.first
+        job.requeue!
+* Start a worker:
+        cyclop -c config.yml
+About
+-----
+License
+-------
+cyclop is Copyright © 2011 TalentBox SA. It is free software, and may be redistributed under the terms specified in the LICENSE file.

data/Rakefile ADDED Viewed

@@ -0,0 +1,59 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+require 'rake/clean'
+# Original Author: Ryan Tomayko
+# Copied from https://github.com/rtomayko/rocco/blob/master/Rakefile
+begin
+  require 'rocco/tasks'
+  Rocco::make 'docs/'
+  desc "Build Cyclop Docs"
+  task :docs => :rocco
+  desc 'Build docs and open in browser for the reading'
+  task :read => :docs do
+    sh 'open docs/lib/cyclop.html'
+  end
+  # Make index.html meta redirect to lib/cyclop.html
+  file 'docs/index.html' do |f|
+    sh %Q{echo '<html><head><meta http-equiv="refresh" content="1;url=http://talentbox.github.com/cyclop/lib/cyclop.html"></head><body></body></html>' > docs/index.html}
+  end
+  task :docs => 'docs/index.html'
+  CLEAN.include 'docs/index.html'
+  # GITHUB PAGES ===============================================================
+  desc 'Update gh-pages branch'
+  task :pages => ['docs/.git', :docs] do
+    rev = `git rev-parse --short HEAD`.strip
+    Dir.chdir 'docs' do
+      sh "git add *.html"
+      sh "git add lib/*.html"
+      sh "git add lib/cyclop/*.html"
+      sh "git commit -m 'rebuild pages from #{rev}'" do |ok,res|
+        if ok
+          verbose { puts "gh-pages updated" }
+          sh "git push -q o HEAD:gh-pages"
+        end
+      end
+    end
+  end
+  # Update the pages/ directory clone
+  file 'docs/.git' => ['docs/', '.git/refs/heads/gh-pages'] do |f|
+    sh "cd docs && git init -q && git remote add o ../.git" if !File.exist?(f.name)
+    sh "cd docs && git fetch -q o && git reset -q --hard o/gh-pages && touch ."
+  end
+  CLOBBER.include 'docs/.git'
+rescue LoadError
+  warn "#$! -- rocco tasks not loaded."
+  task :rocco
+end
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+task :default => :spec

data/bin/cyclop ADDED Viewed

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+require "optparse"
+require "cyclop"
+options = {}
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: cyclop [options]"
+  opts.on("-c", "--config CONFIG", "Read config from CONFIG file") do |c|
+    options[:config] = c
+  end
+  opts.on("-h", "--help", "Display more help") do |h|
+    options[:help] = h
+  end
+  opts.on("-V", "--version", "Display version") do |v|
+    puts Cyclop::VERSION
+    exit
+  end
+end
+opts.parse!
+if options[:config] && !options[:help]
+  unless File.exists?(options[:config]) && File.readable?(options[:config])
+    puts "Config file '#{options[:config]}' do not exists or is not readable."
+    exit(1)
+  end
+  config = nil
+  begin
+    config = YAML::load File.read options[:config]
+  rescue ArgumentError
+  end
+  unless config
+    puts "Config file '#{options[:config]}' format is invalid."
+    exit(1)
+  end
+  begin
+    master = Cyclop::Worker.new config
+  rescue ArgumentError
+    puts $!.message
+  else
+    master.run
+  end
+else
+  puts opts
+  if options[:help]
+    puts <<-EOS
+    Minimal valid YAML config file:
+      mongo:
+        database: "cyclop"
+    Example YAML config file with more options:
+      # Optional: only process jobs from the specified queues
+      queues: [
+        "email", "cache"
+      ]
+      # Load actions in this directory (default to ./actions)
+      actions: "/app/actions"
+      sleep_interval: 0.5 # in seconds
+      log_file: "/var/log/cyclop.log" # will log to STDOUT if missing
+      mongo:
+        database: "cyclop"
+        log: true # for debug only
+        # Without replica sets
+        host: "127.0.0.1" # will default to "127.0.0.1"
+        port: 27017 # will default to 27017
+        # Uncomment the following lines if you use a replica set
+        # hosts: [
+        #   ["127.0.0.1", 27017], ["10.0.0.2", 27017]
+        # ]
+        # rs_name: "production"
+        # read_secondary: true
+    EOS
+  end
+end

data/cyclop.gemspec ADDED Viewed

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "cyclop/version"
+Gem::Specification.new do |s|
+  s.name        = "cyclop"
+  s.version     = Cyclop::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Joseph HALTER", "Jonathan TRON"]
+  s.email       = ["joseph.halter@thetalentbox.com", "jonathan.tron@thetalentbox.com"]
+  s.homepage    = "https://github.com/TalentBox/cyclop"
+  s.summary     = "Job queue with MongoDB"
+  s.description = "Job queue with MongoDB with emphasis on never losing any task even if worker fails hard (segfault)."
+  s.rubyforge_project = "cyclop"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_runtime_dependency("bson_ext", ["~> 1.3.1"])
+  s.add_runtime_dependency("mongo", ["~> 1.3.1"])
+  s.add_runtime_dependency("posix-spawn", ["~> 0.3.6"])
+  s.add_development_dependency("rake", ["~> 0.8.7"])
+  s.add_development_dependency("rspec", ["~> 2.6.0"])
+  s.add_development_dependency("rocco", ["~> 0.7"])
+end

data/lib/cyclop/action.rb ADDED Viewed

@@ -0,0 +1,34 @@
+module Cyclop
+  class Action
+    @@actions = Set.new
+    def self.inherited(klass)
+      @@actions << klass
+    end
+    def self.find_by_queue(queue)
+      actions = @@actions.select{|action| action.queues.include? queue }
+      if @@actions.empty?
+        raise Cyclop::NoActionFound, "No action defined"
+      elsif actions.empty?
+        queues = @@actions.collect(&:queues).flatten.uniq.collect(&:inspect)
+        raise Cyclop::NoActionFound, "No action found for #{queue.inspect} queue. Valid queues: #{queues.join(", ")}"
+      elsif actions.size>1
+        raise Cyclop::ActionQueueClash, "\"#{queue}\" queue belongs to multiple actions: #{actions.collect{|a| a.name}.join(", ")}"
+      else
+        actions.first
+      end
+    end
+    def self.queues
+      []
+    end
+    def self.perform(*args)
+      raise NotImplementedError
+    end
+    def self.to_s
+      "#{name}: #{queues.inspect}"
+    end
+  end
+end

data/lib/cyclop/job.rb ADDED Viewed

@@ -0,0 +1,211 @@
+module Cyclop
+  class Job
+    # Unique identifier
+    attr_accessor :_id
+    # Queue name
+    attr_accessor :queue
+    # Parameters sent to `#perform`
+    attr_accessor :job_params
+    # Delay in seconds
+    attr_accessor :delay
+    # Time until we do start the job
+    attr_accessor :delayed_until
+    # Number of retries before being marked as failed
+    attr_accessor :retries
+    # Time in seconds between retry
+    attr_accessor :splay
+    # Host it's added under
+    attr_accessor :created_by
+    # Time it was created
+    attr_accessor :created_at
+    # Time it was last updated
+    attr_accessor :updated_at
+    # Worker unique identifier
+    attr_accessor :locked_by
+    # Time when worker started
+    attr_accessor :locked_at
+    # Mark as failed
+    attr_accessor :failed
+    # Number of attempts
+    attr_accessor :attempts
+    # Backtraces of unsuccessful attempts
+    attr_accessor :errors
+    def initialize(attrs={})
+      raise ArgumentError, ":queue is required" unless attrs["queue"] || attrs[:queue]
+      self.attributes = attrs
+    end
+    # Create a new job and save it to the queue specified in `opts[:queue]`
+    def self.create(opts={})
+      job = new opts
+      job.save
+      job
+    end
+    # Get the next job from any `opts[:queues]` and mark it as locked
+    def self.next(opts={})
+      raise ArgumentError, "locked_by is required" unless opts[:locked_by]
+      time_now = Time.now.utc
+      conditions = {query: {}}
+      # Not failed jobs only
+      conditions[:query][:failed] = false
+      # Only jobs generated by the specified host if present
+      conditions[:query][:created_by] = opts[:host] if opts[:host]
+      # Skip delayed jobs
+      conditions[:query][:delayed_until] = {"$lte" => time_now}
+      # Filter by queue if present
+      conditions[:query][:queue] = {"$in" => opts[:queues]} if opts[:queues] && !opts[:queues].empty?
+      # Skip locked jobs
+      conditions[:query]["$or"] = [{locked_at: {"$lte" => time_now - 1800}}, {locked_at: nil}]
+      # Last chance to skip dead jobs
+      conditions[:query]["$where"] = "this.attempts <= this.retries"
+      # Set `locked_by` with worker id and increment the number of attempts
+      conditions[:update] = {
+        "$set" => {
+          locked_by: opts[:locked_by],
+          locked_at: time_now,
+        },
+        "$inc" => {
+          attempts: 1
+        }
+      }
+      # Sort by `created_at`
+      conditions[:sort] = [:created_at, :asc]
+      # Returns the modified job
+      conditions[:new] = true
+      new collection.find_and_modify conditions
+    rescue Mongo::OperationFailure
+      nil
+    end
+    # Get failed jobs from any `opts[:queues]`
+    def self.failed(opts={})
+      selector = {}
+      # Failed or dead jobs only
+      selector["$or"] = [
+        {failed: true},
+        {"$where" => "this.attempts > this.retries"},
+      ]
+      # Filter by queue if present
+      selector[:queue] = {"$in" => opts[:queues]} if opts[:queues] && !opts[:queues].empty?
+      options = {}
+      options[:skip] = opts[:skip] if opts[:skip]
+      options[:limit] = opts[:limit] if opts[:limit]
+      collection.find(selector, options).collect{|attrs| new attrs}
+    end
+    def self.find(id)
+      if doc = collection.find_one(id)
+        new doc
+      end
+    end
+    # Save to queue
+    def save
+      self.updated_at = Time.now.utc
+      if persisted?
+        raise NotImplementedError
+      else
+        self.created_at = updated_at
+        self.delayed_until = ::Time.at(created_at.to_i + delay).utc
+        self._id = collection.insert attributes, safe: true
+      end
+      true
+    rescue Mongo::OperationFailure
+      false
+    end
+    def reload
+      self.attributes = collection.find_one _id
+      self
+    end
+    # If we have an id the object is persisted
+    def persisted?
+      !!_id
+    end
+    def ==(other)
+      other._id == _id
+    end
+    # Remove successfully processed job from the queue
+    def complete!
+      collection.remove _id: _id, locked_by: Cyclop.master_id
+    end
+    # Release job for further processing
+    def release!(exception = nil)
+      now = ::Time.at(Time.now.to_i).utc
+      selector = {_id: _id, locked_by: Cyclop.master_id}
+      set = if attempts<=retries
+        {locked_by: nil, locked_at: nil, delayed_until: now+splay}
+      else
+        {failed: true}
+      end
+      update = {"$set" => set}
+      update["$push"] = {
+        :errors => {
+          :locked_by => locked_by,
+          :locked_at => locked_at,
+          :class => exception.class.name,
+          :message => exception.message,
+          :backtrace => exception.backtrace,
+          :created_at => now,
+        },
+      } if exception
+      collection.update selector, update, :safe => true
+    end
+  private
+    def self.collection
+      @@collection ||= Cyclop.db ?
+        Cyclop.db["cyclop_jobs"] : raise(Cyclop::DatabaseNotAvailable)
+    end
+    def collection
+      self.class.collection
+    end
+    def attributes
+      {
+        queue: queue,
+        job_params: job_params,
+        delay: delay,
+        delayed_until: delayed_until,
+        retries: retries,
+        splay: splay,
+        created_by: created_by,
+        created_at: created_at,
+        updated_at: updated_at,
+        locked_by: locked_by,
+        locked_at: locked_at,
+        failed: failed,
+        attempts: attempts,
+        errors: errors,
+      }
+    end
+    def attributes=(attrs)
+      attrs.each do |key, value|
+        send "#{key}=", value
+      end
+      self.delay ||= 0
+      self.retries ||= 0
+      self.splay ||= 60
+      self.created_by ||= Cyclop.host
+      self.failed ||= false
+      self.attempts ||= 0
+      self.errors ||= []
+    end
+  end
+end

data/lib/cyclop/version.rb ADDED Viewed

@@ -0,0 +1,3 @@
+module Cyclop
+  VERSION = "0.1.0"
+end

data/lib/cyclop/worker.rb ADDED Viewed

@@ -0,0 +1,128 @@
+module Cyclop
+  class Worker
+    # Queues to process
+    attr_accessor :queues
+    # Logger for master
+    attr_accessor :logger
+    # How much time to sleep between poll
+    attr_accessor :sleep_interval
+    # Path to actions directory
+    attr_accessor :actions
+    def initialize(config={})
+      raise ArgumentError, 'mongo["database"] is required' unless config["mongo"] && config["mongo"]["database"]
+      self.queues = config["queues"] || []
+      self.logger = Logger.new(config["log_file"] || $stdout)
+      self.sleep_interval = config["sleep_interval"] || 1
+      self.actions = config["actions"] || "./actions"
+      connection = if config["mongo"]["hosts"]
+        Mongo::ReplSetConnection.new(
+          *config["mongo"]["hosts"],
+          rs_name: config["mongo"]["rs_name"],
+          read_secondary: !!config["mongo"]["read_secondary"],
+          logger: (logger if config["mongo"]["log"]),
+        )
+      else
+        Mongo::Connection.new(
+          (config["mongo"]["host"] || "127.0.0.1"),
+          (config["mongo"]["port"] || 27017),
+          logger: (logger if config["mongo"]["log"]),
+        )
+      end
+      Cyclop.db = connection.db config["mongo"]["database"]
+    end
+    # Start processing jobs
+    def run
+      register_signal_handlers
+      loop do
+        if @stop
+          log "Shutting down..."
+          break
+        end
+        if job = next_job
+          @sleeping = false
+          if @pid = fork
+            msg = "Forked process #{@pid} to work on job #{job.queue}-#{job._id}..."
+            log msg
+            procline msg
+            Process.wait
+            log "Child process #{@pid} ended with status: #{$?}"
+            if $?.exitstatus==0
+              job.complete!
+            else
+              job.release!
+            end
+          else
+            procline "Processing #{job.queue}-#{job._id} (started at #{Time.now.utc})"
+            exit! perform job
+          end
+        else
+          log "No more job to process, start sleeping..." unless @sleeping
+          @sleeping = true
+          sleep sleep_interval
+        end
+      end
+    end
+    # Called inside forked process
+    #
+    # Parameters:
+    #
+    # * (Cyclop::Job) job - the job to process
+    #
+    def perform(job)
+      load_actions
+      Cyclop::Action.find_by_queue(job.queue).perform(*job.job_params)
+      0
+    rescue Exception => e
+      log e.to_s
+      job.release! e
+      1
+    end
+    # Gracefull shutdown
+    def stop
+      @stop = true
+    end
+    # Forced shutdown
+    def stop!
+      if @pid
+        Process.kill "TERM", @pid
+        Process.wait
+      end
+      exit!
+    end
+  private
+    # Trap signals
+    #
+    # QUIT - graceful shutdown
+    # INT - first gracefull shutdown, second time force shutdown
+    # TERM - force shutdown
+    def register_signal_handlers
+      trap("QUIT") { stop }
+      trap("INT")  { @stop ? stop! : stop }
+      trap("TERM") { stop! }
+    end
+    def next_job
+      Cyclop.next *queues
+    end
+    def procline(line)
+      $0 = "cyclop-#{Cyclop::VERSION}: #{line}"
+    end
+    def log(message)
+      logger << "#{Time.now}: #{message}\n"
+    end
+    def load_actions
+      Dir["#{actions}/*.rb"].each{|action| require action}
+    end
+  end
+end