threaded_in_memory_queue 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: caa7adac3732ae22522e0e9c76514d76b807c4c4
+  data.tar.gz: 61eb8f9b6e4848a16c36dbb6c630df00c4ec5c97
+SHA512:
+  metadata.gz: 12cff123c6b70a73e98f2d468db8a9161d063ecc4ad93ce483bd3664828b86b19abe87efb113e831f0754a6e854cccf7d085c57134b688fcec03cf71bcdd36a7
+  data.tar.gz: 75309317bc57b8f362a19fe574794e97a1e51eb41b1f257ca023a92827f23b076e89e32493a133f2ce6d69bc7c0f9e44c433d387cbd400213194a8fd9b68e295

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - ruby-head
+  - jruby-19mode
+  - rbx-19mode
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,20 @@
+PATH
+  remote: .
+  specs:
+    threaded_in_memory_queue (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    metaclass (0.0.1)
+    mocha (0.14.0)
+      metaclass (~> 0.0.1)
+    rake (10.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mocha
+  rake
+  threaded_in_memory_queue!

data/README.md

@@ -0,0 +1,97 @@
+## Threaded In Memory Queue
+
+[![Build Status](https://travis-ci.org/schneems/threaded_in_memory_queue.png?branch=master)](https://travis-ci.org/schneems/threaded_in_memory_queue)
+
+
+A simple non-durable in memory queue for running background tasks using threads.
+
+## Why
+
+Projects like [Resque](https://github.com/resque/resque), [delayed job](https://github.com/collectiveidea/delayed_job), [queue classic](https://github.com/ryandotsmith/queue_classic), and [sidekiq](https://github.com/mperham/sidekiq) are great. They use data stores like postgres and redis to store information to be processed later. If you're prototyping a system or don't have access to a data store, you might still want to push off some work to a background process. If that's the case an in-memory threaded queue might be a good fit.
+
+## Install
+
+In your `Gemfile`:
+
+```ruby
+gem 'threaded_in_memory_queue'
+```
+
+Then run `$ bundle install`
+
+## Use it
+
+Add this code in an initializer to start the in memory queue worker (configuration options are below):
+
+```ruby
+ThreadedInMemoryQueue.start
+```
+
+Define your task to be processed:
+
+```ruby
+class Archive
+  def self.call(repo_id, branch = 'master')
+    repo = Repository.find(repo_id)
+    repo.create_archive(branch)
+  end
+end
+```
+
+It can be any object that responds to `call` but we recommend a class or module which makes switching to a durable queue later easier.
+
+Then to enqueue a task to be run in the background use `ThreadedInMemoryQueue.enqueue`:
+
+```ruby
+repo = Repo.last
+ThreadedInMemoryQueue.enqueue(Archive, repo.last, 'staging')
+```
+
+The first argument is a class that defines the task to be processed and the rest of the arguments are passed to the task when it is run.
+
+# Configure
+
+The default number of worker threads is 16, you can configure that when you start your queue:
+
+```ruby
+ThreadedInMemoryQueue.start(size: 5)
+```
+
+By default jobs have a timeout value of 60 seconds. Since this is an in-memory queue (goes away when your process terminates) it is in your best interests to keep jobs small and quick, and not overload the queue. You can configure a different timeout on start:
+
+```ruby
+ThreadedInMemoryQueue.start(timeout: 90) # timeout is in seconds
+```
+
+Want a different logger? Specify a different Logger:
+
+```ruby
+ThreadedInMemoryQueue.start(logger: MyCustomLogger.new)
+```
+
+For testing or guaranteed code execution use the Inline option:
+
+```ruby
+ThreadedInMemoryQueue.inline = true
+```
+
+This option bypasses the queue and executes code as it comes.
+
+## Thread Considerations
+
+This worker operates in the same process as your app, that means if your app is CPU bound, it will not be very useful. This worker uses threads which means that to be useful your app needs to either use IO (database calls, file writes/reads, shelling out, etc.) or run on JRuby or Rubinius.
+
+To make sure all items in your queue are processed you can add a condition `at_exit` to your program:
+
+```ruby
+at_exit do
+  ThreadedInMemoryQueue.stop
+end
+```
+
+This call takes an optional timeout value (in seconds), the default is 10.
+
+## License
+
+MIT
+

data/Rakefile

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+require 'bundler/gem_tasks'
+
+require 'rake'
+require 'rake/testtask'
+
+task :default => [:test]
+
+test_task = Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end

data/lib/threaded_in_memory_queue.rb

@@ -0,0 +1,49 @@
+require 'thread'
+require 'timeout'
+require 'logger'
+
+require 'threaded_in_memory_queue/version'
+require 'threaded_in_memory_queue/inline'
+require 'threaded_in_memory_queue/timeout'
+
+module ThreadedInMemoryQueue
+  extend Inline
+
+  class << self
+    attr_accessor :logger
+  end
+
+  def self.start(options = {})
+    self.logger = options[:logger] if options[:logger]
+    self.master = Master.new(options).start
+    return self
+  end
+
+  def self.master
+    Thread.current[:threaded_in_memory_queue_master]
+  end
+
+  def self.master=(master)
+    Thread.current[:threaded_in_memory_queue_master] = master
+  end
+
+  def self.enqueue(klass, *args)
+    raise NoWorkersError, "must start worker before enqueueing jobs" unless master
+    master.enqueue(klass, *args)
+    return true
+  end
+
+  def self.stop(timeout = 10)
+    return true unless master
+    master.stop(timeout)
+    return true
+  end
+end
+
+ThreadedInMemoryQueue.logger = Logger.new(STDOUT)
+ThreadedInMemoryQueue.logger.level = Logger::INFO
+
+
+require 'threaded_in_memory_queue/errors'
+require 'threaded_in_memory_queue/worker'
+require 'threaded_in_memory_queue/master'

data/lib/threaded_in_memory_queue/errors.rb

@@ -0,0 +1,5 @@
+module ThreadedInMemoryQueue
+  class NoWorkersError < RuntimeError; end
+
+  class WorkerNotStarted < RuntimeError; end
+end

data/lib/threaded_in_memory_queue/inline.rb

@@ -0,0 +1,15 @@
+module ThreadedInMemoryQueue
+  module Inline
+    def inline
+      Thread.current[:threaded_worker_inline]
+    end
+
+    def inline?
+      inline
+    end
+
+    def inline=(inline)
+      Thread.current[:threaded_worker_inline] = inline
+    end
+  end
+end

data/lib/threaded_in_memory_queue/master.rb

@@ -0,0 +1,63 @@
+module ThreadedInMemoryQueue
+  class Master
+    include ThreadedInMemoryQueue::Timeout
+    attr_reader :workers, :logger
+    extend Inline
+
+    DEFAULT_TIMEOUT = 60 # seconds, 1 minute
+    DEFAULT_SIZE    = 16
+
+    def initialize(options = {})
+      @queue   = Queue.new
+      @size    = options[:size]    || DEFAULT_SIZE
+      @timeout = options[:timeout] || DEFAULT_TIMEOUT
+      @logger  = options[:logger]  || ThreadedInMemoryQueue.logger
+      @workers = []
+    end
+
+    def start
+      return self if alive?
+      @size.times { @workers << Worker.new(@queue, timeout: @timeout).start }
+      return self
+    end
+
+    def join
+      workers.each {|w| w.join }
+      return self
+    end
+
+    def poison
+      workers.each {|w| w.poison }
+      return self
+    end
+
+    def stop(timeout = 10)
+      poison
+      timeout(timeout, "waiting for workers to stop") do
+        while self.alive?
+          sleep 0.1
+        end
+        self.join
+      end
+      return self
+    end
+
+    def enqueue(klass, *json)
+      if self.class.inline?
+        klass.call(*json)
+      else
+        raise NoWorkersError, "No workers" unless alive?
+        @queue.enq([klass, json])
+      end
+      return true
+    end
+
+    def alive?
+      return false if workers.empty?
+      workers.detect {|w| w.alive? }
+    end
+  end
+end
+
+
+

data/lib/threaded_in_memory_queue/timeout.rb

@@ -0,0 +1,11 @@
+module ThreadedInMemoryQueue
+  module Timeout
+    def timeout(timeout, message = "", &block)
+      ::Timeout.timeout(timeout) do
+        yield
+      end
+    rescue ::Timeout::Error
+      logger.error("Took longer than #{timeout} to #{message.inspect}")
+    end
+  end
+end

data/lib/threaded_in_memory_queue/version.rb

@@ -0,0 +1,3 @@
+module ThreadedInMemoryQueue
+  VERSION = "0.0.1"
+end

data/lib/threaded_in_memory_queue/worker.rb

@@ -0,0 +1,55 @@
+module ThreadedInMemoryQueue
+  class Worker
+    DEFAULT_TIMEOUT = 60 # seconds, 1 minute
+    POISON          = "poison"
+    include ThreadedInMemoryQueue::Timeout
+    attr_reader :queue, :logger
+
+    def initialize(queue, options = {})
+      @queue   = queue
+      @timeout = options[:timeout] || DEFAULT_TIMEOUT
+      @logger  = options[:logger]  || ThreadedInMemoryQueue.logger
+    end
+
+    def thread
+      raise WorkerNotStarted, "Must start worker before using" unless @thread
+      @thread
+    end
+
+    def start
+      @thread ||= create_thread
+      return self
+    end
+
+    def poison(times = 1)
+      @queue.enq(POISON)
+    end
+
+    def alive?
+      return false unless @thread
+      thread.alive?
+    end
+
+    def join
+      return false unless @thread
+      thread.join
+    end
+
+    private
+    def create_thread
+      Thread.new {
+        logger.info("Worker #{object_id} ready")
+        loop do
+          payload   = queue.pop
+          job, json = *payload
+          break if payload == POISON
+
+          self.timeout(@timeout, "job: #{job.to_s}") do
+            job.call(*json)
+          end
+        end
+        logger.info("Worker #{object_id} stopped")
+      }
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+Bundler.require
+
+require 'threaded_in_memory_queue'
+require 'test/unit'
+require "mocha/setup"
+
+
+module Dummy
+end
+
+ThreadedInMemoryQueue.logger.level = Logger::WARN

data/test/threaded_in_memory_queue/master_test.rb

@@ -0,0 +1,61 @@
+require 'test_helper'
+
+class MasterTest < Test::Unit::TestCase
+
+  def test_thread_worker_creation
+    size   = 1
+    master = ThreadedInMemoryQueue::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+
+    size   = 3
+    master = ThreadedInMemoryQueue::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+
+    size   = 6
+    master = ThreadedInMemoryQueue::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+
+    size   = 16
+    master = ThreadedInMemoryQueue::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+  end
+
+  def test_calls_contents_of_blocks
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    master = ThreadedInMemoryQueue::Master.new(size: 1)
+    master.start
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    master.enqueue(job, 1)
+    master.enqueue(job, 2)
+    master.stop
+  end
+
+  def test_calls_context_of_klass
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    job = Class.new do
+      def self.call(num)
+        Dummy.process(num)
+      end
+    end
+
+    master = ThreadedInMemoryQueue::Master.new(size: 1)
+    master.start
+    master.enqueue(job, 1)
+    master.enqueue(job, 2)
+    master.stop
+  end
+end

data/test/threaded_in_memory_queue/worker_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+class WorkerTest < Test::Unit::TestCase
+  def setup
+    @worker = ThreadedInMemoryQueue::Worker.new(Queue.new, timeout: 1)
+    @worker.start
+  end
+
+  def teardown
+
+  end
+
+  def test_calls_contents_of_blocks
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    @worker.queue << [job, 1]
+    @worker.queue << [job, 2]
+    @worker.poison
+    @worker.join
+  end
+
+  def test_calls_context_of_klass
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    job = Class.new do
+      def self.call(num)
+        Dummy.process(num)
+      end
+    end
+
+    @worker.queue << [job, 1]
+    @worker.queue << [job, 2]
+    @worker.poison
+    @worker.join
+  end
+end

data/test/threaded_in_memory_queue_test.rb

@@ -0,0 +1,56 @@
+require 'test_helper'
+
+class ThreadedInMemoryQueueTest < Test::Unit::TestCase
+
+  def test_calls_contents_of_blocks
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    ThreadedInMemoryQueue.start
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    ThreadedInMemoryQueue.enqueue(job, 1)
+    ThreadedInMemoryQueue.enqueue(job, 2)
+    ThreadedInMemoryQueue.stop
+  end
+
+  def test_calls_contents_of_klasses
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    ThreadedInMemoryQueue.start
+
+    job = Class.new do
+      def self.call(num)
+        Dummy.process(num)
+      end
+    end
+
+    ThreadedInMemoryQueue.enqueue(job, 1)
+    ThreadedInMemoryQueue.enqueue(job, 2)
+    ThreadedInMemoryQueue.stop
+  end
+
+  def test_configure_size
+    size = 1
+    ThreadedInMemoryQueue.start(size: size)
+    assert_equal size, ThreadedInMemoryQueue.master.workers.size
+    ThreadedInMemoryQueue.stop
+
+    size = 3
+    ThreadedInMemoryQueue.start(size: size)
+    assert_equal size, ThreadedInMemoryQueue.master.workers.size
+    ThreadedInMemoryQueue.stop
+
+    size = 6
+    ThreadedInMemoryQueue.start(size: size)
+    assert_equal size, ThreadedInMemoryQueue.master.workers.size
+    ThreadedInMemoryQueue.stop
+
+    size = 16
+    ThreadedInMemoryQueue.start(size: size)
+    assert_equal size, ThreadedInMemoryQueue.master.workers.size
+    ThreadedInMemoryQueue.stop
+  end
+end

data/threaded_in_memory_queue.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'threaded_in_memory_queue/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "threaded_in_memory_queue"
+  gem.version       = ThreadedInMemoryQueue::VERSION
+  gem.authors       = ["Richard Schneeman"]
+  gem.email         = ["richard.schneeman+rubygems@gmail.com"]
+  gem.description   = %q{ Queue stuff in memory }
+  gem.summary       = %q{ Memory, Enqueue stuff you will }
+  gem.homepage      = "https://github.com/schneems/threaded_in_memory_queue"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "mocha"
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: threaded_in_memory_queue
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Richard Schneeman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ' Queue stuff in memory '
+email:
+- richard.schneeman+rubygems@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/threaded_in_memory_queue.rb
+- lib/threaded_in_memory_queue/errors.rb
+- lib/threaded_in_memory_queue/inline.rb
+- lib/threaded_in_memory_queue/master.rb
+- lib/threaded_in_memory_queue/timeout.rb
+- lib/threaded_in_memory_queue/version.rb
+- lib/threaded_in_memory_queue/worker.rb
+- test/test_helper.rb
+- test/threaded_in_memory_queue/master_test.rb
+- test/threaded_in_memory_queue/worker_test.rb
+- test/threaded_in_memory_queue_test.rb
+- threaded_in_memory_queue.gemspec
+homepage: https://github.com/schneems/threaded_in_memory_queue
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Memory, Enqueue stuff you will
+test_files:
+- test/test_helper.rb
+- test/threaded_in_memory_queue/master_test.rb
+- test/threaded_in_memory_queue/worker_test.rb
+- test/threaded_in_memory_queue_test.rb