rack-app-worker 0.1.0

data/lib/rack/app/worker/logger.rb

@@ -0,0 +1,18 @@
+require 'logger'
+class Rack::App::Worker::Logger
+
+  def self.default_out(new_out=nil)
+    @default_out = new_out unless new_out.nil?
+    @default_out ||= $stdout
+  end
+
+  def initialize(out=self.class.default_out)
+    @logger = ::Logger.new(out)
+    @logger.level= Rack::App::Worker::Environment.log_level
+  end
+
+  [:debug, :info, :warn, :error, :fatal, :unknown].each do |severity_level|
+    define_method(severity_level) { |message| @logger.public_send(severity_level, message) }
+  end
+
+end

data/lib/rack/app/worker/observer.rb

@@ -0,0 +1,100 @@
+require 'logger'
+require 'rack/app/worker'
+class Rack::App::Worker::Observer
+
+  def initialize
+    @shutdown_signal_received = false
+    @ready_for_shutdown = false
+  end
+
+  def start
+    logger.info(__method__.to_s)
+    loop do
+      break if shutdown_signal_received
+
+      logger.debug(Rack::App::Worker::Register.worker_definitions.keys.inspect)
+      Rack::App::Worker::Register.worker_definitions.values.each do |definition|
+
+        queue = rabbitmq.send_queue(definition[:name])
+        status = check_status(queue)
+
+        if need_more_worker?(status)
+          create_consumer(definition)
+        end
+
+        if need_less_worker?(status)
+          signal_shutdown_for_a_consumer(definition)
+        end
+
+      end
+
+      sleep(heartbeat_interval)
+
+    end
+  end
+
+  def stop
+    @shutdown_signal_received = true
+    sleep(0.1) until @ready_for_shutdown
+  end
+
+  protected
+
+  def check_status(queue)
+    queue.status
+  rescue Timeout::Error
+    retry
+  end
+
+  def create_consumer(definition)
+    logger.info("#{__method__}(#{definition[:name]})")
+    Rack::App::Worker::Consumer.new(definition).start
+  end
+
+  def signal_shutdown_for_a_consumer(definition)
+    logger.info("#{__method__}(#{definition[:name]})")
+    Rack::App::Worker::Consumer.new(definition).stop
+  end
+
+  def heartbeat_interval
+    Rack::App::Worker::Environment.heartbeat_interval
+  end
+
+  def message_count_limit
+    Rack::App::Worker::Environment.message_count_limit
+  end
+
+  def shutdown_signal_received
+    if @shutdown_signal_received
+      logger.info(__method__.to_s)
+      consumers.each { |c| c.stop_all }
+      rabbitmq.session.close
+      @ready_for_shutdown = true
+    end
+    (!!@ready_for_shutdown)
+  end
+
+  def need_less_worker?(status)
+    (status[:message_count] < message_count_limit) and (status[:consumer_count] > 1)
+  end
+
+  def need_more_worker?(status)
+    ((status[:consumer_count] == 0) or (status[:message_count] >= message_count_limit)) and
+        status[:consumer_count] <= Rack::App::Worker::Environment.max_consumer_number
+  end
+
+  def consumers
+    Rack::App::Worker::Register.worker_definitions.values.map do |definition|
+      Rack::App::Worker::Consumer.new(definition)
+    end
+  end
+
+  def rabbitmq
+    @rabbitmq ||= Rack::App::Worker::RabbitMQ.new
+  end
+
+  def logger
+    @logger ||= Rack::App::Worker::Logger.new
+  end
+
+end

data/lib/rack/app/worker/rabbit_mq.rb

@@ -0,0 +1,117 @@
+require 'bunny'
+require 'rack/app/worker'
+class Rack::App::Worker::RabbitMQ
+
+  def session
+    check_connection
+    @session
+  end
+
+  def channel
+    @channel ||= create_channel
+    @channel = create_channel unless @channel.open?
+    @channel
+  end
+
+  def send_exchange(name)
+    exchange_for('send', name)
+  end
+
+  def broadcast_exchange(name)
+    exchange_for('broadcast', name)
+  end
+
+  def send_queue(name)
+    queue_name = "#{namespace}.#{cluster}.#{name}"
+    queue = channel.queue(queue_name, :durable => true, :auto_delete => false)
+    queue.bind(send_exchange(name)) unless exchange_already_bind?(queue, name)
+    return queue
+  end
+
+  def create_broadcast_queue(name)
+    queue = channel.queue('', :exclusive => true, :auto_delete => false)
+    queue.bind(broadcast_exchange(name)) unless exchange_already_bind?(queue, name)
+    return queue
+  end
+
+  protected
+
+  def exchange_already_bind?(queue, name)
+    queue.recover_bindings.any? { |binding| binding[:exchange] == exchange_name('send', name) }
+  rescue Timeout::Error
+    sleep(rand(1..5))
+    retry
+  end
+
+  def create_new_session
+    session = ::Bunny.new
+    session.start
+    return session
+  end
+
+  def check_connection
+    case @session
+
+      when ::Bunny::Session
+        @session.close if @session.status == :not_connected
+        create_session if @session.closed?
+
+      when NilClass
+        create_session
+
+    end
+  rescue ::Bunny::TCPConnectionFailedForAllHosts
+    sleep(1)
+    retry
+  end
+
+  def create_session
+    new_session = create_new_session
+    new_session.logger.level = Logger::ERROR
+    @session = new_session
+  end
+
+  def cluster
+    Rack::App::Worker::Environment.worker_cluster
+  end
+
+  def namespace
+    Rack::App::Worker::Environment.namespace
+  end
+
+  def create_channel
+    new_channel = session.create_channel
+    new_channel.basic_qos(Rack::App::Worker::Environment.queue_qos)
+    new_channel
+  end
+
+  def exchange_cache
+    @exchange_cache ||= {}
+  end
+
+  def exchange_for(type, name)
+    exchange_cache[name] ||= proc {
+      channel.fanout(exchange_name(type, name), :durable => true)
+    }.call
+  end
+
+  def exchange_name(type, name)
+    "#{namespace}.#{type}.#{name}"
+  end
+
+  def session_finalizer!
+    @session_finalizer ||= lambda do
+      this = self
+      Kernel.at_exit do
+        begin
+          session = this.instance_variable_get(:@session)
+          session && session.respond_to?(:close) && session.close
+        rescue Timeout::Error
+          nil
+        end
+      end
+      true
+    end.call
+  end
+
+end

data/lib/rack/app/worker/register.rb

@@ -0,0 +1,24 @@
+require 'rack/app/worker'
+module Rack::App::Worker::Register
+
+  require 'rack/app/worker/register/builder'
+  require 'rack/app/worker/register/clients'
+
+  extend self
+
+  def add(name,class_constructor)
+    builder = Rack::App::Worker::Register::Builder.new(name.to_sym)
+    builder.consumer(class_constructor)
+    worker_definitions[name.to_sym]= builder.to_def
+    nil
+  end
+
+  def [](name)
+    worker_definitions[name.to_sym]
+  end
+
+  def worker_definitions
+    @worker_definitions ||= {}
+  end
+
+end

data/lib/rack/app/worker/register/builder.rb

@@ -0,0 +1,25 @@
+class Rack::App::Worker::Register::Builder
+
+  def initialize(name)
+    @name = name
+  end
+
+  def consumer(class_definition)
+    if class_definition.is_a?(Class)
+      @consumer_class = class_definition
+    elsif class_definition.is_a?(Proc)
+      klass = Class.new
+      klass.class_exec(&class_definition)
+      @consumer_class = klass
+    end
+  end
+
+  def to_def
+    {
+        class: @consumer_class,
+        name: @name,
+        client: Rack::App::Worker::ClientProxy.new(@name)
+    }
+  end
+
+end

data/lib/rack/app/worker/register/clients.rb

@@ -0,0 +1,10 @@
+require 'rack/app/worker'
+module Rack::App::Worker::Register::Clients
+
+  extend(self)
+
+  def [](name)
+    Rack::App::Worker::Register[name][:client]
+  end
+
+end

data/lib/rack/app/worker/utils.rb

@@ -0,0 +1,18 @@
+module Rack::App::Worker::Utils
+
+  extend(self)
+
+  def process_alive?(pid)
+    ::Process.kill(0, pid.to_i)
+    return true
+  rescue ::Errno::ESRCH
+    return false
+  end
+
+  def maximum_allowed_process_number
+    (`ulimit -u`.to_i * 0.75).to_i + 10
+  rescue Errno::ENOENT
+    100
+  end
+
+end

data/lib/rack/app/worker/version.rb

@@ -0,0 +1,2 @@
+require 'rack/app/worker'
+Rack::App::Worker::VERSION = File.read(File.join(File.dirname(__FILE__), '..', '..', '..', '..', 'VERSION')).strip

data/rack-app-worker.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "rack-app-worker"
+  spec.version       = File.read(File.join(File.dirname(__FILE__),'VERSION')).strip
+  spec.authors       = ["Adam Luzsi"]
+  spec.email         = ["adamluzsi@gmail.com"]
+
+  spec.summary       = %q{Rack::App framework background worker extension}
+  spec.description   = %q{Rack::App framework background worker extension}
+
+  spec.homepage      = "http://www.rack-app.com"
+  spec.license       = 'Apache License Version 2.0'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.0")
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency "rack-app", ">= 3.6.0"
+  spec.add_dependency "bunny", ">= 2.3.0"
+
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: rack-app-worker
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Adam Luzsi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rack-app
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+description: Rack::App framework background worker extension
+email:
+- adamluzsi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- bin/console
+- bin/setup
+- lib/rack/app/worker.rb
+- lib/rack/app/worker/cli.rb
+- lib/rack/app/worker/client_proxy.rb
+- lib/rack/app/worker/client_proxy/wrapper.rb
+- lib/rack/app/worker/consumer.rb
+- lib/rack/app/worker/daemonizer.rb
+- lib/rack/app/worker/dsl.rb
+- lib/rack/app/worker/dsl/for_class.rb
+- lib/rack/app/worker/dsl/for_endpoints.rb
+- lib/rack/app/worker/environment.rb
+- lib/rack/app/worker/logger.rb
+- lib/rack/app/worker/observer.rb
+- lib/rack/app/worker/rabbit_mq.rb
+- lib/rack/app/worker/register.rb
+- lib/rack/app/worker/register/builder.rb
+- lib/rack/app/worker/register/clients.rb
+- lib/rack/app/worker/utils.rb
+- lib/rack/app/worker/version.rb
+- rack-app-worker.gemspec
+homepage: http://www.rack-app.com
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Rack::App framework background worker extension
+test_files: []