threaded 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: db08fff84d920a28793a4f9d73340879fdcbb03c
+  data.tar.gz: 884731a111fbf0c32bcbc3b5ee4e5f105e290566
+SHA512:
+  metadata.gz: bdb4e659e9eab461a2eaf0939b6951f269f42d41d812da326cb4de43336d6633afeb8b6287a82bcf989a31bc4d98c29fea35073bb2ff8da36ba486be9c68435d
+  data.tar.gz: 86e6fc98fbd9245ae59642ffd1832ffdba70abdcbeeb128cf25c7a6b153ec02a296bdf143cf8fbe55cc2e376ee3e7c0f62cd3fe794b4b1d6f5009b48c0543132

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+*.gem

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - ruby-head
+  - jruby-19mode
+  - rbx-19mode
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx-19mode

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,20 @@
+PATH
+  remote: .
+  specs:
+    threaded (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    metaclass (0.0.2)
+    mocha (1.0.0)
+      metaclass (~> 0.0.1)
+    rake (10.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mocha
+  rake
+  threaded!

data/README.md

@@ -0,0 +1,108 @@
+## Threaded
+
+[![Build Status](https://travis-ci.org/schneems/threaded.png?branch=master)](https://travis-ci.org/schneems/threaded)
+
+Simpler than actors, easier than threads. Get threaded!
+
+## Why
+
+Projects like [Resque](https://github.com/resque/resque), [delayed job](https://github.com/collectiveidea/delayed_job), [queue classic](https://github.com/ryandotsmith/queue_classic), and [sidekiq](https://github.com/mperham/sidekiq) are great. They use data stores like postgres and redis to store information to be processed later. If you're prototyping a system or don't have access to a data store, you might still want to push off some work to a background process. If that's the case an in-memory threaded queue might be a good fit.
+
+## Install
+
+In your `Gemfile`:
+
+```ruby
+gem 'threaded'
+```
+
+Then run `$ bundle install`
+
+## Use it
+
+Define your task to be processed:
+
+```ruby
+class Archive
+  def self.call(repo_id, branch = 'master')
+    repo = Repository.find(repo_id)
+    repo.create_archive(branch)
+  end
+end
+```
+
+It can be any object that responds to `call` but we recommend a class or module which makes switching to a durable queue later easier.
+
+Then to enqueue a task to be run in the background use `Threaded.enqueue`:
+
+```ruby
+repo = Repo.last
+Threaded.enqueue(Archive, repo.id, 'staging')
+```
+
+The first argument is a class that defines the task to be processed and the rest of the arguments are passed to the task when it is run.
+
+
+
+# Configure
+
+The default number of worker threads is 16, you can configure that when you start your queue:
+
+```ruby
+Threaded.config do |config|
+  config.size = 5
+end
+```
+
+By default jobs have a timeout value of 60 seconds. Since this is an in-memory queue (goes away when your process terminates) it is in your best interests to keep jobs small and quick, and not overload the queue. You can configure a different timeout on start:
+
+```ruby
+Threaded.config do |config|
+  config.timeout = 90 # timeout is in seconds
+end
+```
+
+Want a different logger? Specify a different Logger:
+
+```ruby
+Threaded.config do |config|
+  config.logger = Logger.new(STDOUT)
+end
+```
+
+As soon as you call `enqueue` a new thread will be started, if you wish to explicitly start all threads you can call `Threaded.start`. You can also inline your config if you want when you start the queue:
+
+```ruby
+Threaded.start(size: 5, timeout: 90, logger: Logger.new(STDOUT))
+```
+
+For testing or guaranteed code execution use the `inline` option:
+
+```ruby
+Threaded.inline = true
+```
+
+This option bypasses the queue and executes code as it comes.
+
+## Thread Considerations
+
+This worker operates in the same process as your app, that means if your app is CPU bound, it will not be very useful. This worker uses threads which means that to be useful your app needs to either use IO (database calls, file writes/reads, shelling out, etc.) or run on JRuby or Rubinius.
+
+To make sure all items in your queue are processed you can add a condition `at_exit` to your program:
+
+```ruby
+at_exit do
+  Threaded.stop
+end
+```
+
+This call takes an optional timeout value (in seconds).
+
+```ruby
+Threaded.stop(42)
+```
+
+## License
+
+MIT
+

data/Rakefile

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+require 'bundler/gem_tasks'
+
+require 'rake'
+require 'rake/testtask'
+
+task :default => [:test]
+
+test_task = Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end

data/lib/threaded.rb

@@ -0,0 +1,79 @@
+require 'thread'
+require 'timeout'
+require 'logger'
+
+require 'threaded/version'
+require 'threaded/timeout'
+
+module Threaded
+  STOP_TIMEOUT = 10 # seconds
+  extend self
+  attr_accessor :inline, :logger, :size, :timeout
+  alias :inline? :inline
+
+  @mutex = Mutex.new
+
+  def start(options = {})
+    self.master = options
+    self.master.start
+    return self
+  end
+
+  def configure(&block)
+    raise "Queue is already started, must configure queue before starting" if started?
+    @mutex.synchronize do
+      yield self
+    end
+  end
+  alias :config  :configure
+
+  def started?
+    return false unless master
+    master.alive?
+  end
+
+  def stopped?
+    !started?
+  end
+
+  def master(options = {})
+    @mutex.synchronize do
+      return @master if @master
+      self.logger  = options[:logger]  if options[:logger]
+      self.size    = options[:size]    if options[:size]
+      self.timeout = options[:timeout] if options[:timeout]
+      @master = Master.new(logger:  self.logger,
+                           size:    self.size,
+                           timeout: self.timeout)
+    end
+  end
+  alias :master= :master
+
+  def later(&block)
+    Threaded::Promise.new(&block).later
+  end
+
+  def enqueue(job, *args)
+    if inline?
+      job.call(*args)
+    else
+      master.enqueue(job, *args)
+    end
+    return true
+  end
+
+  def stop(timeout = STOP_TIMEOUT)
+    return true unless master
+    master.stop(timeout)
+    return true
+  end
+end
+
+Threaded.logger       = Logger.new(STDOUT)
+Threaded.logger.level = Logger::INFO
+
+
+require 'threaded/errors'
+require 'threaded/worker'
+require 'threaded/master'
+require 'threaded/promise'

data/lib/threaded/errors.rb

@@ -0,0 +1,5 @@
+module Threaded
+  class NoWorkersError < RuntimeError; end
+
+  class WorkerNotStarted < RuntimeError; end
+end

data/lib/threaded/master.rb

@@ -0,0 +1,80 @@
+module Threaded
+  class Master
+    include Threaded::Timeout
+    attr_reader :workers, :logger
+
+    DEFAULT_TIMEOUT = 60 # seconds, 1 minute
+    DEFAULT_SIZE    = 16
+
+    def initialize(options = {})
+      @queue    = Queue.new
+      @mutex    = Mutex.new
+      @stopping = false
+      @max      = options[:size]     || DEFAULT_SIZE
+      @timeout  = options[:timeout]  || DEFAULT_TIMEOUT
+      @logger   = options[:logger]   || Threaded.logger
+      @workers  = []
+    end
+
+    def enqueue(job, *json)
+      @queue.enq([job, json])
+
+      new_worker if needs_workers? && @queue.size > 0
+      raise NoWorkersError unless alive?
+      return true
+    end
+
+    def alive?
+      return false if workers.empty?
+      workers.detect {|w| w.alive? }
+    end
+
+    def start
+      return self if alive?
+      @max.times { new_worker }
+      return self
+    end
+
+    def stop(timeout = 10)
+      poison
+      timeout(timeout, "waiting for workers to stop") do
+        while self.alive?
+          sleep 0.1
+        end
+        join
+      end
+      return self
+    end
+
+    def size
+      @workers.size
+    end
+
+    private
+
+    def needs_workers?
+      size < @max
+    end
+
+    def new_worker
+      @mutex.synchronize do
+        return false unless needs_workers?
+        return false if @stopping
+        @workers << Worker.new(@queue, timeout: @timeout)
+      end
+    end
+
+    def join
+      workers.each {|w| w.join }
+      return self
+    end
+
+    def poison
+      @mutex.synchronize do
+        @stopping = true
+      end
+      workers.each {|w| w.poison }
+      return self
+    end
+  end
+end

data/lib/threaded/promise.rb

@@ -0,0 +1,80 @@
+module Threaded
+  class Promise
+    class NoJobError < StandardError
+      def initialize
+        super "No job present for #{self.inspect}"
+      end
+    end
+
+    attr_reader :has_run; alias :has_run? :has_run
+    attr_reader :running; alias :running? :running
+    attr_reader :error
+
+    def initialize(&job)
+      raise "Must supply a job" unless job
+      @mutex   = Mutex.new
+      @has_run = false
+      @running = false
+      @result  = nil
+      @error   = nil
+      @job     = job
+    end
+
+    def later
+      Threaded.enqueue(self)
+      self
+    end
+
+    def call
+      @mutex.synchronize do
+        return true if running? || has_run?
+        begin
+          if @job
+            @running = true
+            @result  = @job.call
+          else
+            raise NoJobError
+          end
+        rescue Exception => error
+          @error   = error
+        ensure
+          @has_run = true
+        end
+      end
+    end
+
+    def now
+      wait_for_it!
+      raise error, error.message, error.backtrace if error
+      @result
+    end
+    alias :join  :now
+    alias :value :now
+
+    private
+    def wait_for_it!
+      return true if has_run?
+
+      if running?
+        @mutex.synchronize {} # waits for lock to be released
+      else
+        call
+      end
+    end
+  end
+end
+
+# job = Threaded.later do
+
+
+# end
+
+# job.now
+
+# job = Threaded::Promise.new
+# job.enqueue do
+
+
+# end
+
+# job.now

data/lib/threaded/timeout.rb

@@ -0,0 +1,11 @@
+module Threaded
+  module Timeout
+    def timeout(timeout, message = "", &block)
+      ::Timeout.timeout(timeout) do
+        yield
+      end
+    rescue ::Timeout::Error
+      logger.error("Took longer than #{timeout} to #{message.inspect}")
+    end
+  end
+end

data/lib/threaded/version.rb

@@ -0,0 +1,3 @@
+module Threaded
+  VERSION = "0.0.1"
+end

data/lib/threaded/worker.rb

@@ -0,0 +1,48 @@
+module Threaded
+  class Worker
+    DEFAULT_TIMEOUT = 60 # seconds, 1 minute
+    POISON          = "poison"
+    include Threaded::Timeout
+    attr_reader :queue, :logger, :thread
+
+    def initialize(queue, options = {})
+      @queue   = queue
+      @timeout = options[:timeout] || DEFAULT_TIMEOUT
+      @logger  = options[:logger]  || Threaded.logger
+      @thread  = create_thread
+    end
+
+    def poison
+      @queue.enq(POISON)
+    end
+
+    def start
+      puts "start is deprecated, thread is started when worker created"
+    end
+
+    def alive?
+      thread.alive?
+    end
+
+    def join
+      thread.join
+    end
+
+    private
+    def create_thread
+      Thread.new {
+        logger.debug("Threaded In Memory Queue Worker '#{object_id}' ready")
+        loop do
+          payload   = queue.pop
+          job, json = *payload
+          break if payload == POISON
+
+          self.timeout(@timeout, "job: #{job.to_s}") do
+            job.call(*json)
+          end
+        end
+        logger.debug("Threaded In Memory Queue Worker '#{object_id}' stopped")
+      }
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+Bundler.require
+
+require 'threaded'
+require 'test/unit'
+require "mocha/setup"
+
+
+module Dummy
+end
+
+Threaded.logger.level = Logger::WARN

data/test/threaded.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+class ThreadedTest < Test::Unit::TestCase
+
+  def test_started?
+    Threaded.start
+    assert Threaded.started?
+    Threaded.stop
+    sleep 1
+    assert Threaded.stopped?
+    refute Threaded.started?
+  end
+
+  def test_inline
+    Dummy.expects(:process).with(1).once
+    job = Proc.new {|x| Dummy.process(x) }
+
+    Threaded.inline = true
+    Threaded.enqueue(job, 1)
+    assert Threaded.inline
+    assert Threaded.stopped?
+  ensure
+    Threaded.inline = false
+  end
+
+  def test_enqueues
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    Threaded.start
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    Threaded.enqueue(job, 1)
+    Threaded.enqueue(job, 2)
+  ensure
+    Threaded.stop
+  end
+
+end

data/test/threaded/config_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+require 'stringio'
+
+class ConfigTest < Test::Unit::TestCase
+
+  def teardown
+    Threaded.stop
+  end
+
+  def test_config_works
+    fake_out = StringIO.new
+    logger   = Logger.new(fake_out)
+    size     = rand(1..99)
+    timeout  = rand(1..99)
+
+    Threaded.configure do |config|
+      config.size    = size
+      config.logger  = logger
+      config.timeout = timeout
+    end
+
+    Threaded.start
+
+    assert_equal size,    Threaded.size
+    assert_equal timeout, Threaded.timeout
+    assert_equal logger,  Threaded.logger
+  end
+
+  def test_config_cannot_call_after_start
+    Threaded.start
+    assert_raise(RuntimeError) do
+      Threaded.configure do |config|
+        config.size    = size
+        config.logger  = logger
+        config.timeout = timeout
+      end
+    end
+  end
+
+end

data/test/threaded/master_test.rb

@@ -0,0 +1,96 @@
+require 'test_helper'
+
+class MasterTest < Test::Unit::TestCase
+
+  def test_non_explicit_thread_start
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    master = Threaded::Master.new(size: 1)
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    master.enqueue(job, 1)
+    master.enqueue(job, 2)
+    master.stop
+  end
+
+  def test_creates_up_to_max_workers
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    master = Threaded::Master.new(size: 2)
+
+    job = Proc.new {|x| sleep 0.5; Dummy.process(x) }
+
+    master.enqueue(job, 1)
+    master.enqueue(job, 2)
+
+    assert_equal 2, master.size
+
+    Dummy.expects(:process).with(3).times(3)
+    master.enqueue(job, 3)
+    master.enqueue(job, 3)
+    master.enqueue(job, 3)
+
+    assert_equal 2, master.size
+    master.stop
+  end
+
+  def test_thread_worker_creation
+    size   = 1
+    master = Threaded::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+
+    size   = 3
+    master = Threaded::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+
+    size   = 6
+    master = Threaded::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+
+    size   = 16
+    master = Threaded::Master.new(size: size)
+    master.start
+    assert_equal size, master.workers.size
+    master.stop
+  end
+
+  def test_calls_contents_of_blocks
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    master = Threaded::Master.new(size: 1)
+    master.start
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    master.enqueue(job, 1)
+    master.enqueue(job, 2)
+    master.stop
+  end
+
+  def test_calls_context_of_klass
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    job = Class.new do
+      def self.call(num)
+        Dummy.process(num)
+      end
+    end
+
+    master = Threaded::Master.new(size: 1)
+    master.start
+    master.enqueue(job, 1)
+    master.enqueue(job, 2)
+    master.stop
+  end
+end

data/test/threaded/worker_test.rb

@@ -0,0 +1,39 @@
+require 'test_helper'
+
+class WorkerTest < Test::Unit::TestCase
+  def setup
+    @worker = Threaded::Worker.new(Queue.new, timeout: 1)
+  end
+
+  def teardown
+
+  end
+
+  def test_calls_contents_of_blocks
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    job = Proc.new {|x| Dummy.process(x) }
+
+    @worker.queue << [job, 1]
+    @worker.queue << [job, 2]
+    @worker.poison
+    @worker.join
+  end
+
+  def test_calls_context_of_klass
+    Dummy.expects(:process).with(1).once
+    Dummy.expects(:process).with(2).once
+
+    job = Class.new do
+      def self.call(num)
+        Dummy.process(num)
+      end
+    end
+
+    @worker.queue << [job, 1]
+    @worker.queue << [job, 2]
+    @worker.poison
+    @worker.join
+  end
+end

data/threaded.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'threaded/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "threaded"
+  gem.version       = Threaded::VERSION
+  gem.authors       = ["Richard Schneeman"]
+  gem.email         = ["richard.schneeman+rubygems@gmail.com"]
+  gem.description   = %q{ Queue stuff in memory }
+  gem.summary       = %q{ Memory, Enqueue stuff you will }
+  gem.homepage      = "https://github.com/schneems/threaded"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake",  "~> 10.1"
+  gem.add_development_dependency "mocha", "~>  1.0"
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: threaded
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Richard Schneeman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: " Queue stuff in memory "
+email:
+- richard.schneeman+rubygems@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/threaded.rb
+- lib/threaded/errors.rb
+- lib/threaded/master.rb
+- lib/threaded/promise.rb
+- lib/threaded/timeout.rb
+- lib/threaded/version.rb
+- lib/threaded/worker.rb
+- test/test_helper.rb
+- test/threaded.rb
+- test/threaded/config_test.rb
+- test/threaded/master_test.rb
+- test/threaded/promise_test.rb
+- test/threaded/worker_test.rb
+- threaded.gemspec
+homepage: https://github.com/schneems/threaded
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Memory, Enqueue stuff you will
+test_files:
+- test/test_helper.rb
+- test/threaded.rb
+- test/threaded/config_test.rb
+- test/threaded/master_test.rb
+- test/threaded/promise_test.rb
+- test/threaded/worker_test.rb
+has_rdoc: