RubyGems - cyclop - Versions diffs - 0.1.0 - Mend

cyclop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (19) hide show

data/.gitignore +7 -0
data/.rspec +2 -0
data/Gemfile +3 -0
data/README.md +88 -0
data/Rakefile +59 -0
data/bin/cyclop +81 -0
data/cyclop.gemspec +29 -0
data/lib/cyclop/action.rb +34 -0
data/lib/cyclop/job.rb +211 -0
data/lib/cyclop/version.rb +3 -0
data/lib/cyclop/worker.rb +128 -0
data/lib/cyclop.rb +131 -0
data/spec/cyclop/action_spec.rb +46 -0
data/spec/cyclop/job_spec.rb +273 -0
data/spec/cyclop/worker_spec.rb +78 -0
data/spec/cyclop_spec.rb +39 -0
data/spec/fixtures/actions/email.rb +14 -0
data/spec/spec_helper.rb +15 -0
metadata +141 -0

data/.gitignore ADDED Viewed

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+docs/
+*.log
+*.yml

data/.rspec ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ --color
2	+ --format documentation

data/Gemfile ADDED Viewed

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+gemspec

data/README.md ADDED Viewed

@@ -0,0 +1,88 @@
+Cyclop
+======
+Job queue with MongoDB with emphasis on never losing any task even if worker fails hard (segfault).
+Dependencies
+------------
+* Ruby >= 1.9.2
+* gem "mongo", "~> 1.3.1"
+* gem "posix-spawn", "~> 0.3.6"
+Usage
+-----
+* Give Cyclop access to mongo:
+        Cyclop.db = Mongo::Connection.new["database_name"]
+    or with Replica-Sets
+        Cyclop.db = Mongo::ReplSetConnection.new["database_name"]
+    or if you're using MongoMapper:
+        Cyclop.db = MongoMapper.database
+    or if you're using Mongoid:
+        Cyclop.db = Mongoid.database
+* Queue a new task:
+        Cyclop.push({
+          queue: :upload,
+          job_params: {
+            url: "http://example.com",
+          },
+        })
+* Queue a new task to process in 5 minutes, to retry 3 times in case of error with a 1 minute delay between each:
+        Cyclop.push({
+          queue: :convert,
+          job_params: {
+            tmp_file: "/tmp/uploaded_file_32.png",
+          },
+          delay: 300,
+          retries: 3,
+          splay: 60,
+        })
+* Get next job:
+        Cyclop.next
+* Get next job on specific queues:
+        Cyclop.next :upload, :convert
+* Get next job on specific queues for a specific host:
+        Cyclop.next :upload, :convert, host: "tartarus.local"
+* Get failed jobs (limit to 30):
+        Cyclop.failed limit: 30
+* Get failed jobs (skip first 10, limit to 30):
+        Cyclop.failed skip: 10, limit: 30
+* Requeue a failed job:
+        job = Cyclop.failed.first
+        job.requeue!
+* Start a worker:
+        cyclop -c config.yml
+About
+-----
+License
+-------
+cyclop is Copyright © 2011 TalentBox SA. It is free software, and may be redistributed under the terms specified in the LICENSE file.

data/Rakefile ADDED Viewed

@@ -0,0 +1,59 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+require 'rake/clean'
+# Original Author: Ryan Tomayko
+# Copied from https://github.com/rtomayko/rocco/blob/master/Rakefile
+begin
+  require 'rocco/tasks'
+  Rocco::make 'docs/'
+  desc "Build Cyclop Docs"
+  task :docs => :rocco
+  desc 'Build docs and open in browser for the reading'
+  task :read => :docs do
+    sh 'open docs/lib/cyclop.html'
+  end
+  # Make index.html meta redirect to lib/cyclop.html
+  file 'docs/index.html' do |f|
+    sh %Q{echo '<html><head><meta http-equiv="refresh" content="1;url=http://talentbox.github.com/cyclop/lib/cyclop.html"></head><body></body></html>' > docs/index.html}
+  end
+  task :docs => 'docs/index.html'
+  CLEAN.include 'docs/index.html'
+  # GITHUB PAGES ===============================================================
+  desc 'Update gh-pages branch'
+  task :pages => ['docs/.git', :docs] do
+    rev = `git rev-parse --short HEAD`.strip
+    Dir.chdir 'docs' do
+      sh "git add *.html"
+      sh "git add lib/*.html"
+      sh "git add lib/cyclop/*.html"
+      sh "git commit -m 'rebuild pages from #{rev}'" do |ok,res|
+        if ok
+          verbose { puts "gh-pages updated" }
+          sh "git push -q o HEAD:gh-pages"
+        end
+      end
+    end
+  end
+  # Update the pages/ directory clone
+  file 'docs/.git' => ['docs/', '.git/refs/heads/gh-pages'] do |f|
+    sh "cd docs && git init -q && git remote add o ../.git" if !File.exist?(f.name)
+    sh "cd docs && git fetch -q o && git reset -q --hard o/gh-pages && touch ."
+  end
+  CLOBBER.include 'docs/.git'
+rescue LoadError
+  warn "#$! -- rocco tasks not loaded."
+  task :rocco
+end
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+task :default => :spec

data/bin/cyclop ADDED Viewed

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+require "optparse"
+require "cyclop"
+options = {}
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: cyclop [options]"
+  opts.on("-c", "--config CONFIG", "Read config from CONFIG file") do |c|
+    options[:config] = c
+  end
+  opts.on("-h", "--help", "Display more help") do |h|
+    options[:help] = h
+  end
+  opts.on("-V", "--version", "Display version") do |v|
+    puts Cyclop::VERSION
+    exit
+  end
+end
+opts.parse!
+if options[:config] && !options[:help]
+  unless File.exists?(options[:config]) && File.readable?(options[:config])
+    puts "Config file '#{options[:config]}' do not exists or is not readable."
+    exit(1)
+  end
+  config = nil
+  begin
+    config = YAML::load File.read options[:config]
+  rescue ArgumentError
+  end
+  unless config
+    puts "Config file '#{options[:config]}' format is invalid."
+    exit(1)
+  end
+  begin
+    master = Cyclop::Worker.new config
+  rescue ArgumentError
+    puts $!.message
+  else
+    master.run
+  end
+else
+  puts opts
+  if options[:help]
+    puts <<-EOS
+    Minimal valid YAML config file:
+      mongo:
+        database: "cyclop"
+    Example YAML config file with more options:
+      # Optional: only process jobs from the specified queues
+      queues: [
+        "email", "cache"
+      ]
+      # Load actions in this directory (default to ./actions)
+      actions: "/app/actions"
+      sleep_interval: 0.5 # in seconds
+      log_file: "/var/log/cyclop.log" # will log to STDOUT if missing
+      mongo:
+        database: "cyclop"
+        log: true # for debug only
+        # Without replica sets
+        host: "127.0.0.1" # will default to "127.0.0.1"
+        port: 27017 # will default to 27017
+        # Uncomment the following lines if you use a replica set
+        # hosts: [
+        #   ["127.0.0.1", 27017], ["10.0.0.2", 27017]
+        # ]
+        # rs_name: "production"
+        # read_secondary: true
+    EOS
+  end
+end

data/cyclop.gemspec ADDED Viewed

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "cyclop/version"
+Gem::Specification.new do |s|
+  s.name        = "cyclop"
+  s.version     = Cyclop::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Joseph HALTER", "Jonathan TRON"]
+  s.email       = ["joseph.halter@thetalentbox.com", "jonathan.tron@thetalentbox.com"]
+  s.homepage    = "https://github.com/TalentBox/cyclop"
+  s.summary     = "Job queue with MongoDB"
+  s.description = "Job queue with MongoDB with emphasis on never losing any task even if worker fails hard (segfault)."
+  s.rubyforge_project = "cyclop"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_runtime_dependency("bson_ext", ["~> 1.3.1"])
+  s.add_runtime_dependency("mongo", ["~> 1.3.1"])
+  s.add_runtime_dependency("posix-spawn", ["~> 0.3.6"])
+  s.add_development_dependency("rake", ["~> 0.8.7"])
+  s.add_development_dependency("rspec", ["~> 2.6.0"])
+  s.add_development_dependency("rocco", ["~> 0.7"])
+end

data/lib/cyclop/action.rb ADDED Viewed

@@ -0,0 +1,34 @@
+module Cyclop
+  class Action
+    @@actions = Set.new
+    def self.inherited(klass)
+      @@actions << klass
+    end
+    def self.find_by_queue(queue)
+      actions = @@actions.select{|action| action.queues.include? queue }
+      if @@actions.empty?
+        raise Cyclop::NoActionFound, "No action defined"
+      elsif actions.empty?
+        queues = @@actions.collect(&:queues).flatten.uniq.collect(&:inspect)
+        raise Cyclop::NoActionFound, "No action found for #{queue.inspect} queue. Valid queues: #{queues.join(", ")}"
+      elsif actions.size>1
+        raise Cyclop::ActionQueueClash, "\"#{queue}\" queue belongs to multiple actions: #{actions.collect{|a| a.name}.join(", ")}"
+      else
+        actions.first
+      end
+    end
+    def self.queues
+      []
+    end
+    def self.perform(*args)
+      raise NotImplementedError
+    end
+    def self.to_s
+      "#{name}: #{queues.inspect}"
+    end
+  end
+end

data/lib/cyclop/job.rb ADDED Viewed

@@ -0,0 +1,211 @@
+module Cyclop
+  class Job
+    # Unique identifier
+    attr_accessor :_id
+    # Queue name
+    attr_accessor :queue
+    # Parameters sent to `#perform`
+    attr_accessor :job_params
+    # Delay in seconds
+    attr_accessor :delay
+    # Time until we do start the job
+    attr_accessor :delayed_until
+    # Number of retries before being marked as failed
+    attr_accessor :retries
+    # Time in seconds between retry
+    attr_accessor :splay
+    # Host it's added under
+    attr_accessor :created_by
+    # Time it was created
+    attr_accessor :created_at
+    # Time it was last updated
+    attr_accessor :updated_at
+    # Worker unique identifier
+    attr_accessor :locked_by
+    # Time when worker started
+    attr_accessor :locked_at
+    # Mark as failed
+    attr_accessor :failed
+    # Number of attempts
+    attr_accessor :attempts
+    # Backtraces of unsuccessful attempts
+    attr_accessor :errors
+    def initialize(attrs={})
+      raise ArgumentError, ":queue is required" unless attrs["queue"] || attrs[:queue]
+      self.attributes = attrs
+    end
+    # Create a new job and save it to the queue specified in `opts[:queue]`
+    def self.create(opts={})
+      job = new opts
+      job.save
+      job
+    end
+    # Get the next job from any `opts[:queues]` and mark it as locked
+    def self.next(opts={})
+      raise ArgumentError, "locked_by is required" unless opts[:locked_by]
+      time_now = Time.now.utc
+      conditions = {query: {}}
+      # Not failed jobs only
+      conditions[:query][:failed] = false
+      # Only jobs generated by the specified host if present
+      conditions[:query][:created_by] = opts[:host] if opts[:host]
+      # Skip delayed jobs
+      conditions[:query][:delayed_until] = {"$lte" => time_now}
+      # Filter by queue if present
+      conditions[:query][:queue] = {"$in" => opts[:queues]} if opts[:queues] && !opts[:queues].empty?
+      # Skip locked jobs
+      conditions[:query]["$or"] = [{locked_at: {"$lte" => time_now - 1800}}, {locked_at: nil}]
+      # Last chance to skip dead jobs
+      conditions[:query]["$where"] = "this.attempts <= this.retries"
+      # Set `locked_by` with worker id and increment the number of attempts
+      conditions[:update] = {
+        "$set" => {
+          locked_by: opts[:locked_by],
+          locked_at: time_now,
+        },
+        "$inc" => {
+          attempts: 1
+        }
+      }
+      # Sort by `created_at`
+      conditions[:sort] = [:created_at, :asc]
+      # Returns the modified job
+      conditions[:new] = true
+      new collection.find_and_modify conditions
+    rescue Mongo::OperationFailure
+      nil
+    end
+    # Get failed jobs from any `opts[:queues]`
+    def self.failed(opts={})
+      selector = {}
+      # Failed or dead jobs only
+      selector["$or"] = [
+        {failed: true},
+        {"$where" => "this.attempts > this.retries"},
+      ]
+      # Filter by queue if present
+      selector[:queue] = {"$in" => opts[:queues]} if opts[:queues] && !opts[:queues].empty?
+      options = {}
+      options[:skip] = opts[:skip] if opts[:skip]
+      options[:limit] = opts[:limit] if opts[:limit]
+      collection.find(selector, options).collect{|attrs| new attrs}
+    end
+    def self.find(id)
+      if doc = collection.find_one(id)
+        new doc
+      end
+    end
+    # Save to queue
+    def save
+      self.updated_at = Time.now.utc
+      if persisted?
+        raise NotImplementedError
+      else
+        self.created_at = updated_at
+        self.delayed_until = ::Time.at(created_at.to_i + delay).utc
+        self._id = collection.insert attributes, safe: true
+      end
+      true
+    rescue Mongo::OperationFailure
+      false
+    end
+    def reload
+      self.attributes = collection.find_one _id
+      self
+    end
+    # If we have an id the object is persisted
+    def persisted?
+      !!_id
+    end
+    def ==(other)
+      other._id == _id
+    end
+    # Remove successfully processed job from the queue
+    def complete!
+      collection.remove _id: _id, locked_by: Cyclop.master_id
+    end
+    # Release job for further processing
+    def release!(exception = nil)
+      now = ::Time.at(Time.now.to_i).utc
+      selector = {_id: _id, locked_by: Cyclop.master_id}
+      set = if attempts<=retries
+        {locked_by: nil, locked_at: nil, delayed_until: now+splay}
+      else
+        {failed: true}
+      end
+      update = {"$set" => set}
+      update["$push"] = {
+        :errors => {
+          :locked_by => locked_by,
+          :locked_at => locked_at,
+          :class => exception.class.name,
+          :message => exception.message,
+          :backtrace => exception.backtrace,
+          :created_at => now,
+        },
+      } if exception
+      collection.update selector, update, :safe => true
+    end
+  private
+    def self.collection
+      @@collection ||= Cyclop.db ?
+        Cyclop.db["cyclop_jobs"] : raise(Cyclop::DatabaseNotAvailable)
+    end
+    def collection
+      self.class.collection
+    end
+    def attributes
+      {
+        queue: queue,
+        job_params: job_params,
+        delay: delay,
+        delayed_until: delayed_until,
+        retries: retries,
+        splay: splay,
+        created_by: created_by,
+        created_at: created_at,
+        updated_at: updated_at,
+        locked_by: locked_by,
+        locked_at: locked_at,
+        failed: failed,
+        attempts: attempts,
+        errors: errors,
+      }
+    end
+    def attributes=(attrs)
+      attrs.each do |key, value|
+        send "#{key}=", value
+      end
+      self.delay ||= 0
+      self.retries ||= 0
+      self.splay ||= 60
+      self.created_by ||= Cyclop.host
+      self.failed ||= false
+      self.attempts ||= 0
+      self.errors ||= []
+    end
+  end
+end

data/lib/cyclop/version.rb ADDED Viewed

@@ -0,0 +1,3 @@
+module Cyclop
+  VERSION = "0.1.0"
+end

data/lib/cyclop/worker.rb ADDED Viewed

@@ -0,0 +1,128 @@
+module Cyclop
+  class Worker
+    # Queues to process
+    attr_accessor :queues
+    # Logger for master
+    attr_accessor :logger
+    # How much time to sleep between poll
+    attr_accessor :sleep_interval
+    # Path to actions directory
+    attr_accessor :actions
+    def initialize(config={})
+      raise ArgumentError, 'mongo["database"] is required' unless config["mongo"] && config["mongo"]["database"]
+      self.queues = config["queues"] || []
+      self.logger = Logger.new(config["log_file"] || $stdout)
+      self.sleep_interval = config["sleep_interval"] || 1
+      self.actions = config["actions"] || "./actions"
+      connection = if config["mongo"]["hosts"]
+        Mongo::ReplSetConnection.new(
+          *config["mongo"]["hosts"],
+          rs_name: config["mongo"]["rs_name"],
+          read_secondary: !!config["mongo"]["read_secondary"],
+          logger: (logger if config["mongo"]["log"]),
+        )
+      else
+        Mongo::Connection.new(
+          (config["mongo"]["host"] || "127.0.0.1"),
+          (config["mongo"]["port"] || 27017),
+          logger: (logger if config["mongo"]["log"]),
+        )
+      end
+      Cyclop.db = connection.db config["mongo"]["database"]
+    end
+    # Start processing jobs
+    def run
+      register_signal_handlers
+      loop do
+        if @stop
+          log "Shutting down..."
+          break
+        end
+        if job = next_job
+          @sleeping = false
+          if @pid = fork
+            msg = "Forked process #{@pid} to work on job #{job.queue}-#{job._id}..."
+            log msg
+            procline msg
+            Process.wait
+            log "Child process #{@pid} ended with status: #{$?}"
+            if $?.exitstatus==0
+              job.complete!
+            else
+              job.release!
+            end
+          else
+            procline "Processing #{job.queue}-#{job._id} (started at #{Time.now.utc})"
+            exit! perform job
+          end
+        else
+          log "No more job to process, start sleeping..." unless @sleeping
+          @sleeping = true
+          sleep sleep_interval
+        end
+      end
+    end
+    # Called inside forked process
+    #
+    # Parameters:
+    #
+    # * (Cyclop::Job) job - the job to process
+    #
+    def perform(job)
+      load_actions
+      Cyclop::Action.find_by_queue(job.queue).perform(*job.job_params)
+      0
+    rescue Exception => e
+      log e.to_s
+      job.release! e
+      1
+    end
+    # Gracefull shutdown
+    def stop
+      @stop = true
+    end
+    # Forced shutdown
+    def stop!
+      if @pid
+        Process.kill "TERM", @pid
+        Process.wait
+      end
+      exit!
+    end
+  private
+    # Trap signals
+    #
+    # QUIT - graceful shutdown
+    # INT - first gracefull shutdown, second time force shutdown
+    # TERM - force shutdown
+    def register_signal_handlers
+      trap("QUIT") { stop }
+      trap("INT")  { @stop ? stop! : stop }
+      trap("TERM") { stop! }
+    end
+    def next_job
+      Cyclop.next *queues
+    end
+    def procline(line)
+      $0 = "cyclop-#{Cyclop::VERSION}: #{line}"
+    end
+    def log(message)
+      logger << "#{Time.now}: #{message}\n"
+    end
+    def load_actions
+      Dir["#{actions}/*.rb"].each{|action| require action}
+    end
+  end
+end