toro 0.0.1 → 0.0.2

data/lib/generators/toro/install/install_generator.rb

@@ -0,0 +1,25 @@
+require 'rails/generators'
+require 'rails/generators/base'
+
+module Toro
+  module Generators
+    class InstallGenerator < ::Rails::Generators::Base
+      include ::Rails::Generators::Migration
+      source_root File.expand_path('../templates', __FILE__)
+      desc "Install the migrations"
+
+      def self.next_migration_number(path)
+        unless @prev_migration_nr
+          @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+        else
+          @prev_migration_nr += 1
+        end
+        @prev_migration_nr.to_s
+      end
+
+      def install_migrations
+        migration_template "create_toro_jobs.rb", "db/migrate/create_toro_jobs.rb"
+      end
+    end 
+  end
+end

data/lib/generators/toro/install/templates/create_toro_jobs.rb

@@ -0,0 +1,9 @@
+class CreateToroJobs < ActiveRecord::Migration
+  def self.up
+    Toro::Database.up
+  end
+
+  def self.down
+    Toro::Database.down
+  end
+end

data/lib/tasks/tasks.rb

@@ -0,0 +1,19 @@
+task :environment
+
+namespace :toro do
+  desc "Start a new worker for the (default or $QUEUE) queue"
+  task :start  => :environment do
+    cli = Toro::CLI.new
+    cli.run
+  end
+
+  desc "Setup Toro tables and functions in database"
+  task :up => :environment do
+    Toro::Database.up
+  end
+
+  desc "Remove Toro tables and functions from database."
+  task :down => :environment do
+    Toro::Database.down
+  end
+end

data/lib/toro.rb

@@ -1,4 +1,54 @@
+# Dependencies
+require 'active_support/all'
+require 'active_record'
+require 'celluloid'
+require 'nested-hstore'
+require 'socket'
+
+# Self
+directory = File.dirname(File.absolute_path(__FILE__))
 require "#{directory}/toro/version.rb"
+Dir.glob("#{directory}/toro/*.rb") { |file| require file }
+Dir.glob("#{directory}/toro/middleware/**/*.rb") { |file| require file }
+Dir.glob("#{directory}/generators/**/*.rb") { |file| require file }
 
 module Toro
-end
+  DEFAULTS = {
+    default_queue: 'default',
+    graceful_shutdown_time: 1,
+    hard_shutdown_time: 8,
+    listen_interval: 5
+  }
+
+  class << self
+    def options
+      @options ||= DEFAULTS.dup
+    end
+
+    def options=(options)
+      @options = options
+    end
+
+    def configure_server
+      yield self
+    end
+
+    def server_middleware
+      @server_chain ||= Processor.default_middleware
+      yield @server_chain if block_given?
+      @server_chain
+    end
+
+    def process_identity
+      @process_identity ||= "#{Socket.gethostname}:#{Process.pid}"
+    end
+
+    def logger
+      Toro::Logging.logger
+    end
+
+    def logger=(log)
+      Toro::Logging.logger = log
+    end
+  end
+end

data/lib/toro/actor.rb

@@ -0,0 +1,8 @@
+module Toro
+  module Actor
+    def self.included(klass)
+      klass.__send__(:include, Celluloid)
+      klass.__send__(:task_class, Celluloid::TaskThread)
+    end
+  end
+end

data/lib/toro/actor_manager.rb

@@ -0,0 +1,11 @@
+module Toro
+  module ActorManager
+    def register_actor(key, proxy)
+      actors[key] = proxy
+    end
+
+    def actors
+      @actors ||= {}
+    end
+  end
+end

data/lib/toro/cli.rb

@@ -0,0 +1,87 @@
+module Toro
+  class CLI
+    attr_reader :options
+
+    def initialize(arguments=ARGV)
+      @options = arguments_to_options(arguments)
+      @manager = Manager.new(@options)
+    end
+
+    def run
+      Toro.logger.info 'Starting processing (press Control-C to stop)'
+
+      read_io, write_io = IO.pipe
+      
+      %w(INT TERM USR1 USR2 TTIN).each do |signal|
+        begin
+          trap signal do
+            write_io.puts(signal)
+          end
+        rescue ArgumentError
+          Toro.logger.debug "Signal #{signal} not supported"
+        end
+      end
+
+      begin
+        @manager.start
+
+        while readable_io = IO.select([read_io])
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        Toro.logger.info 'Shutting down...'
+        @manager.stop
+      end
+
+      # Explicitly exit so busy Processor threads can't block
+      # process shutdown.
+      exit 0
+    end
+
+    protected
+
+    def arguments_to_options(arguments)
+      options = {}
+      OptionParser.new do |opts|
+        opts.on('-q', '--queue NAME', 'Queue') { |v| options[:queues] ||= []; options[:queues] << parse_queue(v) }
+        opts.on('-c', '--concurrency CONCURRENCY', 'Concurrency') { |v| options[:concurrency] = Integer(v) }
+      end.parse!(arguments)
+      options
+    end
+
+    def parse_queue(value)
+      value.strip
+    end
+
+    def handle_signal(signal)
+      Toro.logger.debug "Got #{signal} signal"
+      case signal
+      when 'INT'
+        # Handle Ctrl-C in JRuby like MRI
+        # http://jira.codehaus.org/browse/JRUBY-4637
+        raise Interrupt
+      when 'TERM'
+        # Heroku sends TERM and then waits 10 seconds for process to exit.
+        raise Interrupt
+      when 'USR1'
+        Toro.logger.info "Received USR1, no longer accepting new work"
+        @manager.async.stop
+      when 'USR2'
+        if Toro.options[:logfile]
+          Toro.logger.info "Received USR2, reopening log file"
+          Toro::Logging.reopen_logs
+        end
+      when 'TTIN'
+        Thread.list.each do |thread|
+          Toro.logger.info "Thread T#{thread.object_id.to_s(36)} #{thread['label']}"
+          if thread.backtrace
+            Toro.logger.info thread.backtrace.join("\n")
+          else
+            Toro.logger.info "<no backtrace available>"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toro/client.rb

@@ -0,0 +1,17 @@
+module Toro
+  class Client
+    class << self
+      def create_job(item)
+        item.stringify_keys!
+        job_attributes = item_to_job_attributes(item)
+        Job.create!(job_attributes)
+      end
+
+      private
+
+      def item_to_job_attributes(item)
+        { 'status' => 'queued' }.merge(item)
+      end
+    end
+  end
+end

data/lib/toro/database.rb

@@ -0,0 +1,38 @@
+module Toro
+  class Database
+    SQL_DIRECTORY = Pathname.new(File.expand_path('sql', File.dirname(__FILE__)))
+
+    class << self
+      def up
+        execute_file('up')
+      end
+
+      def down
+        execute_file('down')
+      end
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def raw_connection
+        connection.raw_connection
+      end
+
+      def query(sql, parameters=[])
+        raw_connection.exec(sql, parameters)
+      end
+
+      def with_connection(&block)
+        ActiveRecord::Base.connection_pool.with_connection(&block)
+      end
+
+      private
+
+      def execute_file(file_name)
+        file_path = SQL_DIRECTORY.join("#{file_name}.sql")
+        connection.execute(File.read(file_path))
+      end
+    end
+  end
+end

data/lib/toro/fetcher.rb

@@ -0,0 +1,40 @@
+module Toro
+  class Fetcher
+    include Actor
+
+    def initialize(options={})
+      defaults = {
+        queues: [Toro.options[:default_queue]]
+      }
+      options.reverse_merge!(defaults)
+      @queues = options[:queues]
+      @manager = options[:manager]
+      raise 'No manager provided' if @manager.blank?
+    end
+
+    def notify
+      if @manager.is_ready?
+        job = retrieve
+        @manager.assign(job) if job
+      end
+    end
+
+    def fetch
+      job = retrieve
+      @manager.async.assign(job) if job
+    end
+
+    def retrieve
+      job = nil
+      queue_list = @queues.map { |queue| "'#{queue}'" }.join(', ')
+      sql = "SELECT * FROM toro_pop(ARRAY[#{queue_list}]::TEXT[], '#{Toro.process_identity}')"
+      result = nil
+      Toro::Database.with_connection do
+        result = Toro::Database.query(sql).first
+        result = nil if result['id'].nil?
+      end
+      return nil if result.nil?
+      Job.instantiate(result)
+    end
+  end
+end

data/lib/toro/job.rb

@@ -0,0 +1,38 @@
+module Toro
+  class Job < ActiveRecord::Base
+    if ActiveRecord::VERSION::MAJOR < 4 || ActiveRecord.constants.include?(:MassAssignmentSecurity)
+      attr_accessible :queue, :class_name, :args, :name, :created_at, :scheduled_at, :started_at, :finished_at,
+        :status, :started_by, :properties
+    end
+
+    serialize :args
+    serialize :properties, ActiveRecord::Coders::NestedHstore
+
+    self.table_name_prefix = 'toro_'
+    
+    STATUSES = [
+      'queued',
+      'running',
+      'complete',
+      'failed',
+      'scheduled'
+    ]
+
+    class << self
+      def statuses
+        STATUSES
+      end
+    end
+
+    def set_properties(hash)
+      self.properties ||= {}
+      hash.each do |key, value|
+        self.properties[key.to_s] = value
+      end
+    end
+
+    def to_s
+      "Toro::Job ##{id}"
+    end
+  end
+end

data/lib/toro/listener.rb

@@ -0,0 +1,44 @@
+module Toro
+  class Listener
+    include Actor
+
+    def initialize(options={})
+      defaults = {
+        queues: [Toro.options[:default_queue]]
+      }
+      options.reverse_merge!(defaults)
+      @queues = options[:queues]
+      @fetcher = options[:fetcher]
+      @manager = options[:manager]
+      @is_done = false
+      raise 'No fetcher provided' if @fetcher.blank?
+      raise 'No manager provided' if @manager.blank?
+    end
+
+    def start
+      @manager.register_actor(:listener, self)
+      Toro::Database.with_connection do
+        Toro::Database.raw_connection.async_exec(channels.map { |channel| "LISTEN #{channel}" }.join('; '))
+        wait_for_notify
+      end
+    end
+
+    def stop
+      Toro::Database.raw_connection.async_exec(channels.map { |channel| "UNLISTEN #{channel}" }.join('; '))
+      @is_done = true
+    end
+
+    protected
+
+    def wait_for_notify
+      Toro::Database.raw_connection.wait_for_notify(Toro.options[:listen_interval]) do |channel, pid, payload|
+        @fetcher.notify
+      end
+      wait_for_notify unless @is_done
+    end
+
+    def channels
+      @queues.map { |queue| "toro_#{queue}" }
+    end
+  end
+end

data/lib/toro/logging.rb

@@ -0,0 +1,80 @@
+module Toro
+  module Logging
+    class Formatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "[#{time.utc.iso8601} P-#{Process.pid} T-#{Thread.current.object_id.to_s(36)}] #{severity} -- Toro: #{message}\n"
+      end
+    end
+
+    class << self
+      def initialize_logger(log_target=nil)
+        if log_target.nil?
+          log_target = $TESTING ? '/dev/null' : STDOUT
+        end
+
+        old_logger = defined?(@logger) ? @logger : nil
+        @logger = Logger.new(log_target)
+        @logger.level = $TESTING ? Logger::DEBUG : Logger::INFO
+        @logger.formatter = Formatter.new
+        old_logger.close if old_logger && !$TESTING # don't want to close testing's STDOUT logging
+        Celluloid.logger = @logger
+        @logger
+      end
+
+      def logger
+        defined?(@logger) ? @logger : initialize_logger
+      end
+
+      def logger=(log)
+        @logger = (log ? log : Logger.new('/dev/null'))
+        Celluloid.logger = @logger
+        @logger
+      end
+
+      # This reopens ALL logfiles in the process that have been rotated
+      # using logrotate(8) (without copytruncate) or similar tools.
+      # A +File+ object is considered for reopening if it is:
+      #   1) opened with the O_APPEND and O_WRONLY flags
+      #   2) the current open file handle does not match its original open path
+      #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+      # Returns the number of files reopened
+      def reopen_logs
+        to_reopen = []
+        append_flags = File::WRONLY | File::APPEND
+
+        ObjectSpace.each_object(File) do |fp|
+          begin
+            if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+              to_reopen << fp
+            end
+          rescue IOError, Errno::EBADF
+          end
+        end
+
+        nr = 0
+        to_reopen.each do |fp|
+          orig_st = begin
+            fp.stat
+          rescue IOError, Errno::EBADF
+            next
+          end
+
+          begin
+            b = File.stat(fp.path)
+            next if orig_st.ino == b.ino && orig_st.dev == b.dev
+          rescue Errno::ENOENT
+          end
+
+          begin
+            File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
+            fp.sync = true
+            nr += 1
+          rescue IOError, Errno::EBADF
+            # not much we can do...
+          end
+        end
+        nr
+      end
+    end
+  end
+end