toiler 0.1.5 → 0.2.0

data/celluloid-task-pooledfiber/Rakefile

@@ -1,8 +0,0 @@
-require 'bundler/gem_tasks'
-require 'rspec/core/rake_task'
-
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rspec_opts = ['--color --format documentation']
-end
-task default: :spec

data/celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec

@@ -1,18 +0,0 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-Gem::Specification.new do |spec|
-  spec.name          = 'celluloid-task-pooledfiber'
-  spec.version       = '0.2.0'
-  spec.authors       = ['Chris Heald']
-  spec.email         = ['cheald@gmail.com']
-
-  spec.summary       = 'An alternate Task implementation for Celluloid which improves performance by reusing a pool of fibers to run tasks'
-  spec.description   = 'An alternate Task implementation for Celluloid which improves performance by reusing a pool of fibers to run tasks'
-  spec.homepage      = 'http://github.com/cheald/celluloid-task-pooledfiber'
-  spec.license       = 'MIT'
-
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|examples|spec|features)/}) }
-  spec.require_paths = ['lib']
-end

data/celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb

@@ -1,26 +0,0 @@
-require 'celluloid' unless defined? Celluloid
-require 'thread'
-
-require 'celluloid/util/fiber_pool'
-
-module Celluloid
-  class Task
-    # Tasks with a pooled Fiber backend
-    class PooledFiber < Fibered
-      def self.fiber_pool
-        @fiber_pool ||= Util::FiberPool.new
-      end
-
-      def create(&block)
-        queue = Thread.current[:celluloid_queue]
-        actor_system = Thread.current[:celluloid_actor_system]
-        @fiber = PooledFiber.fiber_pool.acquire do
-          Thread.current[:celluloid_role] = :actor
-          Thread.current[:celluloid_queue] = queue
-          Thread.current[:celluloid_actor_system] = actor_system
-          block.call
-        end
-      end
-    end
-  end
-end

data/celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb

@@ -1,95 +0,0 @@
-module Celluloid
-  module Util
-    class FiberPool
-      attr_accessor :stats
-
-      def initialize(trim_size = 64)
-        @trim_size = trim_size
-        @pool = {}
-        @stats = { created: 0, acquired: 0, trimmed: 0, sweep_counter: 0, terminated: 0, terminated_threads: 0, sweeps: 0 }
-        @mutex = Mutex.new
-      end
-
-      def acquire(&block)
-        trim
-        sweep
-        @stats[:acquired] += 1
-        fiber = fiber_pool.shift || create_fiber
-        fiber.resume(block)
-        fiber
-      end
-
-      private
-
-      def create_fiber
-        pool = fiber_pool
-        @stats[:created] += 1
-        Fiber.new do |blk|
-          loop do
-            # At this point, we have a pooled fiber ready for Celluloid to call #resume on.
-            Fiber.yield
-
-            # Once Celluloid resumes the fiber, we need to execute the block the task was
-            # created with. This may call suspend/resume; that's fine.
-            blk.call
-
-            # Once Celluloid completes the task, we release this Fiber back to the pool
-            pool << Fiber.current
-
-            # ...and yield for the next #acquire call.
-            blk = Fiber.yield
-            break if blk == :terminate
-          end
-        end
-      end
-
-      # If the fiber pool has grown too much, shift off and discard
-      # extra fibers so they may be GC'd. This isn't ideal, but it
-      # should guard against runaway resource consumption.
-      def trim
-        pool = fiber_pool
-        while pool.length > @trim_size
-          @stats[:trimmed] += 1
-          pool.shift.resume(:terminate)
-        end
-      end
-
-      def sweep
-        @mutex.synchronize do
-          @stats[:sweep_counter] += 1
-          if @stats[:sweep_counter] > 10_000
-            alive = []
-            Thread.list.each do |thread|
-              alive << thread.object_id if thread.alive? && @pool.key?(thread.object_id)
-            end
-
-            (@pool.keys - alive).each do |thread_id|
-              @pool[thread_id].each do |_fiber|
-                @stats[:terminated] += 1
-                # We can't resume the fiber here because we might resume cross-thread
-                # TODO: How do we deal with alive fibers in a dead thread?
-                # fiber.resume(:terminate)
-              end
-              @stats[:terminated_threads] += 1
-              @pool.delete thread_id
-            end
-            @stats[:sweep_counter] = 0
-            @stats[:sweeps] += 1
-          end
-        end
-      end
-
-      # Fiber pool for this thread. Fibers can't cross threads, so we have to maintain a
-      # pool per thread.
-      #
-      # We keep our pool in an instance variable rather than in thread locals so that we
-      # can sweep out old Fibers from dead threads. This keeps live Fiber instances in
-      # a thread local from keeping the thread from being GC'd.
-      def fiber_pool
-        @mutex.synchronize do
-          @pool[Thread.current.object_id] ||= []
-        end
-      end
-    end
-  end
-end

data/celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb

@@ -1,5 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Celluloid::Task::PooledFiber, actor_system: :within do
-  it_behaves_like 'a Celluloid Task'
-end

data/celluloid-task-pooledfiber/spec/spec_helper.rb

@@ -1,60 +0,0 @@
-$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-
-require 'rubygems'
-require 'bundler/setup'
-
-require 'celluloid/test'
-require 'celluloid/rspec'
-
-require 'celluloid/tasks/pooled_fiber'
-
-$CELLULOID_DEBUG = true
-
-require 'celluloid/probe'
-
-Dir['./spec/support/*.rb'].map { |f| require f }
-
-RSpec.configure do |config|
-  config.filter_run focus: true
-  config.run_all_when_everything_filtered = true
-
-  config.around do |ex|
-    Celluloid.actor_system = nil
-    Thread.list.each do |thread|
-      next if thread == Thread.current
-      if defined?(JRUBY_VERSION)
-        # Avoid disrupting jRuby's "fiber" threads.
-        next if /Fiber/ =~ thread.to_java.getNativeThread.get_name
-      end
-      thread.kill
-    end
-
-    ex.run
-  end
-
-  config.around actor_system: :global do |ex|
-    Celluloid.boot
-    ex.run
-    Celluloid.shutdown
-  end
-
-  config.around actor_system: :within do |ex|
-    Celluloid::ActorSystem.new.within do
-      ex.run
-    end
-  end
-
-  config.mock_with :rspec do |mocks|
-    mocks.verify_doubled_constant_names = true
-    mocks.verify_partial_doubles = true
-  end
-
-  config.around(:each) do |example|
-    example.run
-  end
-
-  # Must be *after* the around hook above
-  require 'rspec/retry'
-  config.verbose_retry = true
-  config.default_sleep_interval = 3
-end

data/celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb

@@ -1,49 +0,0 @@
-class MockActor
-  attr_reader :tasks
-
-  def initialize
-    @tasks = []
-  end
-
-  def setup_thread
-  end
-end
-
-RSpec.shared_examples 'a Celluloid Task' do
-  let(:task_type)     { :foobar }
-  let(:suspend_state) { :doing_something }
-  let(:actor)         { MockActor.new }
-
-  subject { Celluloid.task_class.new(task_type, {}) { Celluloid::Task.suspend(suspend_state) } }
-
-  before :each do
-    Thread.current[:celluloid_actor_system] = Celluloid.actor_system
-    Thread.current[:celluloid_actor] = actor
-  end
-
-  after :each do
-    Thread.current[:celluloid_actor] = nil
-    Thread.current[:celluloid_actor_system] = nil
-  end
-
-  it 'begins with status :new' do
-    expect(subject.status).to be :new
-  end
-
-  it 'resumes' do
-    expect(subject).to be_running
-    subject.resume
-    expect(subject.status).to eq(suspend_state)
-    subject.resume
-    expect(subject).not_to be_running
-  end
-
-  it 'raises exceptions outside' do
-    task = Celluloid.task_class.new(task_type, {}) do
-      fail 'failure'
-    end
-    expect do
-      task.resume
-    end.to raise_exception('failure')
-  end
-end

data/lib/toiler/core_ext.rb

@@ -1,47 +0,0 @@
-begin
-  require 'active_support/core_ext/hash/keys'
-  require 'active_support/core_ext/hash/deep_merge'
-rescue LoadError
-  class Hash
-    def stringify_keys
-      each_key do |key|
-        self[key.to_s] = delete(key)
-      end
-      self
-    end unless {}.respond_to?(:stringify_keys)
-
-    def symbolize_keys
-      each_key do |key|
-        self[(key.to_sym rescue key) || key] = delete(key)
-      end
-      self
-    end unless {}.respond_to?(:symbolize_keys)
-
-    def deep_symbolize_keys
-      each_key do |key|
-        value = delete(key)
-        self[(key.to_sym rescue key) || key] = value
-
-        value.deep_symbolize_keys if value.is_a? Hash
-      end
-      self
-    end unless {}.respond_to?(:deep_symbolize_keys)
-  end
-end
-
-begin
-  require 'active_support/core_ext/string/inflections'
-rescue LoadError
-  class String
-    def constantize
-      names = split('::')
-      names.shift if names.empty? || names.first.empty?
-
-      constant = Object
-      names.each do |name|
-        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-      end
-      constant
-    end
-  end unless ''.respond_to?(:constantize)
-end

data/lib/toiler/environment_loader.rb

@@ -1,82 +0,0 @@
-require 'erb'
-require 'yaml'
-
-module Toiler
-  class EnvironmentLoader
-    attr_reader :options
-
-    def self.load(options)
-      new(options).load
-    end
-
-    def self.load_for_rails_console
-      load(config_file: (Rails.root + 'config' + 'toiler.yml'))
-    end
-
-    def initialize(options)
-      @options = options
-    end
-
-    def load
-      initialize_logger
-      load_rails if options[:rails]
-      require_workers if options[:require]
-      Toiler.options.merge!(config_file_options)
-      Toiler.options.merge!(options)
-      initialize_aws
-    end
-
-    private
-
-    def config_file_options
-      if (path = options[:config_file])
-        unless File.exist?(path)
-          Toiler.logger.warn "Config file #{path} does not exist"
-          path = nil
-        end
-      end
-
-      return {} unless path
-
-      YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
-    end
-
-    def initialize_aws
-      # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-      # when not explicit supplied
-      fail 'AWS Credentials needed!' if Toiler.options[:aws].empty? && (ENV['AWS_ACCESS_KEY_ID'].nil? || ENV['AWS_SECRET_ACCESS_KEY'].nil?)
-      return if Toiler.options[:aws].empty?
-
-      ::Aws.config[:region] = Toiler.options[:aws][:region]
-      ::Aws.config[:credentials] = ::Aws::Credentials.new Toiler.options[:aws][:access_key_id], Toiler.options[:aws][:secret_access_key]
-    end
-
-    def initialize_logger
-      Toiler::Logging.initialize_logger(options[:logfile]) if options[:logfile]
-      Toiler.logger.level = Logger::DEBUG if options[:verbose]
-    end
-
-    def load_rails
-      # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
-
-      require 'rails'
-      if ::Rails::VERSION::MAJOR < 4
-        require File.expand_path('config/environment.rb')
-        ::Rails.application.eager_load!
-      else
-        # Painful contortions, see 1791 for discussion
-        require File.expand_path('config/application.rb')
-        ::Rails::Application.initializer 'toiler.eager_load' do
-          ::Rails.application.config.eager_load = true
-        end
-        require File.expand_path('config/environment.rb')
-      end
-
-      Toiler.logger.info 'Rails environment loaded'
-    end
-
-    def require_workers
-      require options[:require]
-    end
-  end
-end

data/lib/toiler/fetcher.rb

@@ -1,56 +0,0 @@
-module Toiler
-  class Fetcher
-    include Celluloid
-    include Celluloid::Internals::Logger
-
-    FETCH_LIMIT = 10.freeze
-
-    attr_accessor :queue, :wait, :batch, :condition
-
-    finalizer :shutdown
-
-    def initialize(queue, client)
-      debug "Initializing Fetcher for queue #{queue}..."
-      @condition = Celluloid::Condition.new
-      @queue = Queue.new queue, client
-      @wait = Toiler.options[:wait] || 20
-      @batch = Toiler.worker_class_registry[queue].batch?
-      async.poll_messages
-      debug "Finished initializing Fetcher for queue #{queue}"
-    end
-
-    def shutdown
-      debug "Fetcher #{queue.name} shutting down..."
-      instance_variables.each { |iv| remove_instance_variable iv }
-    end
-
-    def processor_finished
-      @condition.broadcast
-    end
-
-    def wait_for_available_processors
-      manager = Toiler.manager
-      @condition.wait if manager.dead? || manager.free_processors(queue) == 0
-    end
-
-    def poll_messages
-      # AWS limits the batch size by 10
-      options = {
-        message_attribute_names: %w(All),
-        wait_time_seconds: wait
-      }
-
-      loop do
-        while (count = Toiler.manager.free_processors(queue.name)) == 0
-          wait_for_available_processors
-        end
-        options[:max_number_of_messages] = (batch || count > FETCH_LIMIT) ? FETCH_LIMIT : count
-        debug "Fetcher #{queue.name} retreiving messages with options: #{options.inspect}..."
-        msgs = queue.receive_messages options
-        debug "Fetcher #{queue.name} retreived #{msgs.count} messages..."
-        next if msgs.empty?
-        Toiler.manager.assign_messages queue.name, msgs
-      end
-    end
-  end
-end