geary 0.0.1

data/lib/gearman/worker.rb

@@ -0,0 +1,61 @@
+require 'celluloid'
+require 'gearman/connection'
+require 'gearman/packet'
+require 'uri'
+
+module Gearman
+  class Worker
+    include Celluloid
+
+    trap_exit :reconnect
+    finalizer :disconnect
+
+    def initialize(address)
+      @address = URI(address)
+      configure_connection Connection.method(:new_link)
+    end
+
+    def can_do(ability)
+      @connection.write(Packet::CAN_DO.new(function_name: ability))
+    end
+
+    def pre_sleep
+      @connection.write(Packet::PRE_SLEEP.new)
+      @connection.next(Packet::NOOP)
+    end
+
+    def grab_job
+      @connection.write(Packet::GRAB_JOB.new)
+      @connection.next(Packet::JOB_ASSIGN, Packet::NO_JOB)
+    end
+
+    def work_exception(handle, data)
+      @connection.write(Packet::WORK_EXCEPTION.new(handle: handle, data: data))
+    end
+
+    def work_complete(handle, data)
+      @connection.write(Packet::WORK_COMPLETE.new(handle: handle, data: data))
+    end
+
+    def disconnect
+      if @connection
+        @connection.terminate if @connection.alive?
+      end
+    end
+
+    def build_connection
+      @connection = @connect.call(@address)
+    end
+
+    def reconnect(*_)
+      disconnect
+      build_connection
+    end
+
+    def configure_connection(connection_routine)
+      @connect = connection_routine
+      reconnect
+    end
+
+  end
+end

data/lib/geary.rb

@@ -0,0 +1,8 @@
+require 'geary/worker'
+
+module Geary
+end
+
+if defined?(Rails)
+  require 'geary/railtie'
+end

data/lib/geary/cli.rb

@@ -0,0 +1,86 @@
+require 'socket'
+
+require 'celluloid'
+
+require 'geary/option_parser'
+require 'geary/manager'
+
+module Geary
+
+  class CLI
+
+    Shutdown = Class.new(StandardError) unless defined? Shutdown
+
+    attr_reader :configuration, :internal_signal_queue, :external_signal_queue
+
+    def initialize(argv, stdout = STDOUT, stderr = STDERR, kernel = Kernel,
+                   pipe = IO.pipe)
+      @argv = argv
+      @stdout = stdout
+      @stdout = stderr
+      @kernel = kernel
+      @internal_signal_queue, @external_signal_queue = pipe
+      @configuration = OptionParser.new.parse(@argv)
+    end
+
+    def execute!
+      Celluloid.logger.level = configuration.log_level
+
+      %w(INT TERM).each do |signal|
+        trap signal do
+          external_signal_queue.puts(signal)
+        end
+      end
+
+      munge_environment_given(configuration)
+      load_rails
+
+      manager = Manager.new(configuration: configuration)
+      manager.start
+
+      begin
+        loop do
+          IO.select([internal_signal_queue])
+          signal = internal_signal_queue.gets.strip
+
+          handle(signal)
+        end
+      rescue Shutdown
+        manager.async.stop
+
+        manager.wait(:done)
+
+        @kernel.exit(0)
+      end
+    end
+
+    def handle(signal)
+      if %w(INT TERM).include?(signal)
+        raise Shutdown
+      end
+    end
+
+    private
+
+    def munge_environment_given(configuration)
+      $:.concat(configuration.included_paths)
+      configuration.required_files.each { |file| require file }
+    end
+
+    def load_rails
+      begin
+        require 'rails'
+      rescue LoadError
+        Celluloid.logger.debug "Unable to load Rails"
+      else
+        Celluloid.logger.debug "Loading Rails"
+
+        require 'geary/railtie'
+        require 'config/environment'
+
+        ::Rails.application.eager_load!
+      end
+    end
+
+  end
+end

data/lib/geary/configuration.rb

@@ -0,0 +1,18 @@
+require 'logger'
+require 'virtus'
+require 'virtus/uri'
+
+module Geary
+  class Configuration
+    include Virtus
+
+    attribute :server_addresses, Array[URI], default: ['gearman://localhost:4730']
+    attribute :concurrency, Integer, default: 25
+    attribute :included_paths, Array, default: %w(.)
+    attribute :required_files, Array, default: []
+    attribute :failure_monitor_interval, Integer, default: 1
+    attribute :jitter, Float, default: 0.01
+    attribute :log_level, Object, default: Logger::INFO
+
+  end
+end

data/lib/geary/error.rb

@@ -0,0 +1,7 @@
+require 'nestegg'
+
+module Geary
+  class Error < StandardError
+    include Nestegg::NestingException
+  end
+end

data/lib/geary/manager.rb

@@ -0,0 +1,84 @@
+require 'celluloid'
+require 'geary/error'
+require 'geary/performer'
+
+module Geary
+  class Manager
+    include Celluloid
+
+    UnexpectedRestart = Class.new(Error) unless defined? UnexpectedRestart
+
+    attr_reader :configuration, :performers
+
+    trap_exit :performer_crashed
+
+    def initialize(options = {})
+      @configuration = options.fetch(:configuration)
+      @performer_type = options.fetch(:performer_type, Performer)
+      @performers = []
+      @crashes = []
+      @server_addresses_by_performer = {}
+    end
+
+    def start
+      async.monitor_crashes
+
+      configuration.server_addresses.each do |server_address|
+        configuration.concurrency.times do
+          start_performer(server_address)
+        end
+      end
+    end
+
+    def stop
+      @performers.select(&:alive?).each(&:terminate)
+
+      after(0) { signal(:done) }
+    end
+
+    private
+
+    def monitor_crashes
+      every(configuration.failure_monitor_interval) do 
+        @crashes.reject! do |server_address|
+          momentarily { start_performer(server_address) }
+        end
+      end
+    end
+
+    def performer_crashed(performer, reason)
+      if String(reason).size > 0
+        forget_performer(performer) do |server_address|
+          @crashes.unshift(server_address)
+        end
+      end
+    end
+
+    def forget_performer(performer, &wants_server_address)
+      _id = performer.object_id
+
+      @performers.delete(performer) do
+        raise UnexpectedRestart, "we don't know about Performer #{_id}"
+      end
+
+      server_address = @server_addresses_by_performer.delete(_id)
+
+      wants_server_address.call(server_address)
+    end
+
+    def start_performer(server_address)
+      performer = @performer_type.new_link(server_address)
+
+      @performers << performer
+      @server_addresses_by_performer[performer.object_id] = server_address
+
+      performer.async.start
+    end
+
+    def momentarily(&action)
+      after(rand + configuration.jitter, &action)
+      true
+    end
+
+  end
+end

data/lib/geary/option_parser.rb

@@ -0,0 +1,42 @@
+require 'optparse'
+
+require 'geary/configuration'
+
+module Geary
+  class OptionParser
+
+    def parse(args)
+      Configuration.new.tap do |configuration|
+        parser_which_configures(configuration).parse!(Array(args))
+      end
+    end
+
+    def parser_which_configures(configuration)
+      ::OptionParser.new do |parser|
+        parser.on('-s', '--server SERVERS', Array) do |server_addresses|
+          configuration.server_addresses = server_addresses
+        end
+
+        parser.on('-r', '--require FILES', Array) do |files|
+          configuration.required_files = files
+        end
+
+        parser.on('-I', '--include PATHS', Array) do |paths|
+          configuration.included_paths = paths
+        end
+
+        parser.on('-c', '--concurrency NUMBER', 'number of concurrent tasks to run per server') do |number|
+          configuration.concurrency = Integer(number)
+        end
+
+        parser.on('-l', '--level LOG_LEVEL', 'log level (FATAL|ERROR|WARN|INFO|DEBUG)') do |level|
+          begin
+            configuration.log_level = Logger.const_get(String(level).upcase)
+          rescue NameError
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/geary/performer.rb

@@ -0,0 +1,73 @@
+require 'celluloid'
+require 'forwardable'
+require 'gearman/worker'
+require 'json'
+
+module Geary
+
+  class Performer
+    include Celluloid
+    extend Forwardable
+
+    finalizer :disconnect
+    trap_exit :reconnect
+
+    def initialize(address)
+      @address = address
+      configure_connection Gearman::Worker.method(:new_link)
+    end
+
+    def start
+      @gearman.can_do('Geary.default')
+
+      loop do
+        packet = @gearman.grab_job
+
+        case packet
+        when Gearman::Packet::JOB_ASSIGN
+          perform(packet)
+        when Gearman::Packet::NO_JOB
+          @gearman.pre_sleep
+        else
+          break
+        end
+      end
+    end
+
+    def perform(packet)
+      job = JSON.parse(packet.data)
+      job_result = nil
+
+      begin
+        worker = ::Object.const_get(job['class']).new
+
+        job_result = worker.perform(*job['args'])
+      rescue => error
+        @gearman.async.work_exception(packet.handle, error.message)
+      else
+        @gearman.async.work_complete(packet.handle, job_result)
+      end
+    end
+
+    def disconnect
+      if @gearman
+        @gearman.terminate if @gearman.alive?
+      end
+    end
+
+    def reconnect(actor, reason)
+      disconnect
+      build_connection
+    end
+
+    def build_connection
+      @gearman = @connect.call(@address)
+    end
+
+    def configure_connection(connection_routine)
+      @connect = connection_routine
+      reconnect(current_actor, nil)
+    end
+
+  end
+end

data/lib/geary/railtie.rb

@@ -0,0 +1,9 @@
+module Geary
+  class Railtie < Rails::Railtie
+
+    config.before_initialize do
+      Rails.application.paths.add 'app/workers', eager_load: true
+    end
+
+  end
+end

data/lib/geary/worker.rb

@@ -0,0 +1,49 @@
+require 'json'
+require 'gearman/client'
+require 'geary/error'
+
+module Geary
+  module Worker
+
+    def perform_async(*args)
+      payload = payload_for(args)
+
+      operation do |gearman|
+        gearman.submit_job_bg('Geary.default', payload.to_json)
+      end
+    end
+
+    protected
+
+    def use_gearman_client(*args)
+      @gearman_client = Gearman::Client.new(*args)
+    end
+
+    def gearman_client
+      @gearman_client || use_gearman_client('gearman://localhost:4730')
+    end
+
+    def payload_for(args)
+      payload = { class: self.to_s, args: args }
+    end
+
+    def operation(&block)
+      attempts = 0
+      failure_threshold = 1
+
+      begin
+        block.call(gearman_client)
+      rescue
+        attempts += 1
+
+        if attempts > failure_threshold
+          raise Error
+        else
+          gearman_client.reconnect
+          retry
+        end
+      end
+    end
+
+  end
+end