ost 0.1.6 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d56949a251817fd42e7b1c1ea085185512f1567
-  data.tar.gz: e4a06723f0629316b924db731ce1d2521e18eccf
+  metadata.gz: d0c37dcb37d2a4a8083505a4cda90d414ae46ba4
+  data.tar.gz: 5fb22633c7b154355b487a764227b2566a5925a2
 SHA512:
-  metadata.gz: a284c35a61569f0cbffb1ef1dc13d71e61c841c582eeda66a2b9e44d6f7d6a76db4ce36a08d2c669b063648ee5a3f70e607cfa6899d684e14e6aba24a0884cb3
-  data.tar.gz: 65d1e5942ff81729e87e9ca30bc163483e7768cfd27daedd3a0bcd6a2b0bf63b8cd06f9240f49180f38173994027034650191fa38880706bec6f1bdab63557f2
+  metadata.gz: f58ea6507fd16c105124a5fcfd5d8865b81154e23a8d2f111b8e92824e71cdeeb3ab0968d97c6bae5ef82f79e0eb6dbcde6ed054cd0bc52db677d7b0a4898bb5
+  data.tar.gz: 6e1184ac0c347eff90229108f384e561248b90636bb0f408e53eb21d0c4670863d5d2298c174077ffa618678f4bfeded253746a3c5347cb388113f849df4e6ec

data/.gems

@@ -0,0 +1,3 @@
+cutest -v 1.2.1
+nido -v 0.0.1
+redic -v 1.1.0

data/.gitignore

@@ -0,0 +1 @@
+/pkg

data/CHANGELOG.md

@@ -0,0 +1,67 @@
+## 1.0.0.rc1
+
+* Use [redic][redic] instead of `redis-rb` as the Redis client. This changes
+  how to connect to a Redis database:
+
+  Before:
+
+  ```ruby
+  Ost.connect(localhost: 6379, port: 6380, db: 2)
+  ```
+
+  Now:
+
+  ```ruby
+  Ost.redis = Redic.new("redis://127.0.0.1:6379/2")
+  ```
+
+## 0.1.6
+
+* Fix bug with non-integer timeouts
+
+* Add `Ost::Queue#size` method. Returns the current size of the queue.
+
+  Example:
+
+  ```ruby
+  Ost[:events].size # => 0
+  Ost[:events].push(1)
+  Ost[:events].size # => 1
+  ```
+
+## 0.1.5
+
+* Fix bug with redis connection handling. Previously, it was just using the default connection
+  of `localhost:6379`, regardless if you setup your app with an `Ost.connect`.
+
+## 0.1.4
+
+* No changes.
+
+## 0.1.3
+
+* Don't yield the block on timeouts.
+
+## 0.1.2
+
+* Change default timeout to 2 seconds.
+
+## 0.1.1
+
+* Use `backup.lpop` instead of `backup.del`.
+
+## 0.1.0
+
+* `Ost#each` no longer rescues exceptions for you.
+
+    You are in charge of rescuing and deciding what to do.
+
+* You can inspect the status of the queue by calling `Ost::Queue#items`.
+
+* If you need access to the underlying Redis key, it's in `Ost::Queue#key`.
+
+* Ost now uses `BRPOPLPUSH` and maintains a backup queue while working.
+
+    You can access this queue using `Ost::Queue#backup`.
+
+[redic]: https://github.com/amakawa/redic

data/README.md

@@ -31,18 +31,34 @@ end
 Usage
 -----
 
-**Ost** connects to Redis automatically with the default options
-(localhost:6379, database 0).
+Ost uses a lightweight Redis client called [Redic][redic]. To connect to
+a Redis database, you will need to set an instance of `Redic`, with a URL
+of the form `redis://:<passwd>@<host>:<port>/<db>`.
 
-You can customize the connection by calling `connect`:
+You can customize the connection by calling `Ost.redis=`:
 
 ``` ruby
-Ost.connect port: 6380, db: 2
+require "ost"
+
+Ost.redis = Redic.new("redis://127.0.0.1:6379")
 ```
 
 Then you only need to refer to a queue for it to pop into existence:
 
 ``` ruby
+require "ost"
+
+Ost.redis = Redic.new("redis://127.0.0.1:6379")
+
+Ost[:rss_feeds] << @feed.id
+```
+
+Ost defaults to a Redic connection to `redis://127.0.0.1:6379`. The example
+above could be rewritten as:
+
+``` ruby
+require "ost"
+
 Ost[:rss_feeds] << @feed.id
 ```
 
@@ -57,7 +73,7 @@ end
 ```
 
 It will pop items from the queue as soon as they become available. It
-uses BRPOPLPUSH with a timeout that can be specified with the
+uses `BRPOPLPUSH` with a timeout that can be specified with the
 `OST_TIMEOUT` environment variable.
 
 Note that in these examples we are pushing numbers to the queue. As
@@ -68,12 +84,6 @@ pop.
 Available methods
 =================
 
-`Ost.connect`: configure the connection to Redis. By default, it
-connects to localhost in port 6379 and uses the database 0. It accepts
-the same options as [redis-rb](https://github.com/redis/redis-rb).
-
-`Ost.stop`: halt processing for all queues.
-
 `Ost[:example].push item`, `Ost[:some_queue] << item`: add `item` to
 the `:example` queue.
 
@@ -81,6 +91,8 @@ the `:example` queue.
 }`: consume `item` from the `:example` queue. If the block doesn't
 complete successfully, the item will be left at a backup queue.
 
+`Ost.stop`: halt processing for all queues.
+
 `Ost[:example].stop`: halt processing for the `example` queue.
 
 Failures
@@ -131,28 +143,4 @@ Installation
 
     $ gem install ost
 
-License
--------
-
-Copyright (c) 2010, 2011, 2012 Michel Martens
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
+[redic]: https://github.com/amakawa/redic

data/lib/ost.rb

@@ -1,34 +1,33 @@
-require "nest"
+require "nido"
+require "redic"
 
 module Ost
-  TIMEOUT = (ENV["OST_TIMEOUT"] || 2).to_i
+  TIMEOUT = ENV["OST_TIMEOUT"] || "2"
 
   class Queue
     attr :key
     attr :backup
 
     def initialize(name)
-      @key = Nest.new(:ost, redis)[name]
+      @key = Nido.new(:ost)[name]
       @backup = @key[Socket.gethostname][Process.pid]
+      @stopping = false
     end
 
     def push(value)
-      key.lpush(value)
+      redis.call("LPUSH", @key, value)
     end
 
     def each(&block)
-      @stopping = false
-
       loop do
-        break if @stopping
-
-        item = @key.brpoplpush(@backup, TIMEOUT)
+        item = redis.call("BRPOPLPUSH", @key, @backup, TIMEOUT)
 
-        next unless item
+        if item
+          block.call(item)
+          redis.call("LPOP", @backup)
+        end
 
-        block.call(item)
-
-        @backup.lpop
+        break if @stopping
       end
     end
 
@@ -37,18 +36,22 @@ module Ost
     end
 
     def items
-      key.lrange(0, -1)
+      redis.call("LRANGE", @key, 0, -1)
     end
 
     alias << push
     alias pop each
 
     def size
-      key.llen
+      redis.call("LLEN", @key)
     end
 
     def redis
-      @redis ||= Redis.connect(Ost.options)
+      defined?(@redis) ? @redis : Ost.redis
+    end
+
+    def redis=(redis)
+      @redis = redis
     end
   end
 
@@ -64,13 +67,11 @@ module Ost
     @queues.each { |_, queue| queue.stop }
   end
 
-  @options = nil
-
-  def self.connect(options = {})
-    @options = options
+  def self.redis
+    @redis ||= Redic.new
   end
 
-  def self.options
-    @options || {}
+  def self.redis=(redis)
+    @redis = redis
   end
 end

data/makefile

@@ -0,0 +1,4 @@
+.PHONY: test
+
+test:
+	cutest ./test/*.rb

data/ost.gemspec

@@ -1,21 +1,16 @@
 Gem::Specification.new do |s|
   s.name              = "ost"
-  s.version           = "0.1.6"
+  s.version           = "1.0.0.rc1"
   s.summary           = "Redis based queues and workers."
   s.description       = "Ost lets you manage queues and workers with Redis."
   s.authors           = ["Michel Martens"]
   s.email             = ["michel@soveran.com"]
   s.homepage          = "http://github.com/soveran/ost"
+  s.license           = "MIT"
 
-  s.files = Dir[
-    "LICENSE",
-    "README.md",
-    "Rakefile",
-    "lib/**/*.rb",
-    "*.gemspec",
-    "test/*.*"
-  ]
+  s.files = `git ls-files`.split("\n")
 
-  s.add_dependency "nest", "~> 1.0"
-  s.add_development_dependency "cutest", "~> 1.0"
+  s.add_dependency "redic"
+  s.add_dependency "nido"
+  s.add_development_dependency "cutest"
 end

data/test/access.rb

@@ -0,0 +1,25 @@
+require_relative "helper"
+
+scope do
+  test "access to underlying key" do
+    assert_equal "ost:events", Ost[:events].key
+  end
+
+  test "access to queued items" do
+    Ost[:events].push(1)
+
+    assert_equal ["1"], Ost[:events].items
+  end
+
+  test "access to queue size" do
+    queue = Ost[:events]
+    assert_equal 0, queue.size
+
+    queue.push(1)
+    assert_equal 1, queue.size
+
+    queue.stop
+    queue.each { }
+    assert_equal 0, queue.size
+  end
+end

data/test/backup.rb

@@ -0,0 +1,38 @@
+require_relative "helper"
+
+scope do
+  test "returns backup key" do
+    hostname = Socket.gethostname
+    pid = Process.pid
+
+    assert_equal "ost:events:#{hostname}:#{pid}", Ost[:events].backup
+  end
+
+  test "maintains a backup queue for when worker dies" do
+    queue = Ost[:events]
+
+    queue.push(1)
+
+    assert_equal 0, queue.redis.call("LLEN", queue.backup)
+
+    begin
+      queue.stop
+      queue.each { |item| item.some_error }
+    rescue
+    end
+
+    assert_equal ["1"], queue.redis.call("LRANGE", queue.backup, 0, -1)
+  end
+
+  test "cleans up the backup queue on success" do
+    queue = Ost[:events]
+    queue.push(1)
+
+    queue.stop
+    queue.each do |item|
+      assert_equal [item], queue.redis.call("LRANGE", queue.backup, 0, -1)
+    end
+
+    assert_equal [], queue.redis.call("LRANGE", queue.backup, 0, -1)
+  end
+end

data/test/connection.rb

@@ -0,0 +1,16 @@
+require_relative "helper"
+
+scope do
+  test "inherits ost redis connection by default" do
+    queue = Ost[:events]
+
+    assert_equal Ost.redis.url, queue.redis.url
+  end
+
+  test "queue can define its own connection" do
+    queue = Ost[:people]
+    queue.redis = Redic.new("redis://localhost:6379/1")
+
+    assert Ost.redis.url != queue.redis.url
+  end
+end

data/test/core.rb

@@ -0,0 +1,17 @@
+require_relative "helper"
+
+scope do
+  test "process items from the queue" do
+    Ost[:events].push(1)
+
+    results = []
+
+    Ost[:events].stop
+    Ost[:events].each do |item|
+      results << item
+    end
+
+    assert_equal [], Ost[:events].items
+    assert_equal ["1"], results
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+require "cutest"
+require_relative "../lib/ost"
+
+Ost.redis = Redic.new("redis://127.0.0.1:6379")
+
+prepare do
+  Ost.redis.call("FLUSHALL")
+end

data/test/stop.rb

@@ -0,0 +1,22 @@
+require_relative "helper"
+
+scope do
+  test "stop processing a queue" do
+    Ost[:events].push(1)
+    Ost[:events].stop
+    Ost[:events].each { }
+
+    assert(true)
+  end
+
+  test "stop processing all queues" do
+    Ost[:events].push(1)
+    Ost[:people].push(1)
+
+    Ost.stop
+    Ost[:events].each { }
+    Ost[:people].each { }
+
+    assert(true)
+  end
+end

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: ost
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Michel Martens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-06 00:00:00.000000000 Z
+date: 2014-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nest
+  name: redic
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nido
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: cutest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 description: Ost lets you manage queues and workers with Redis.
 email:
 - michel@soveran.com
@@ -45,15 +59,23 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .gems
+- .gitignore
+- CHANGELOG.md
 - LICENSE
 - README.md
-- Rakefile
 - lib/ost.rb
+- makefile
 - ost.gemspec
-- test/ost_test.rb
-- test/test_helper.rb
+- test/access.rb
+- test/backup.rb
+- test/connection.rb
+- test/core.rb
+- test/helper.rb
+- test/stop.rb
 homepage: http://github.com/soveran/ost
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -66,9 +88,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.3

data/Rakefile

@@ -1,6 +0,0 @@
-task :test do
-  require "cutest"
-  Cutest.run(Dir["test/ost*"])
-end
-
-task :default => :test

data/test/ost_test.rb

@@ -1,160 +0,0 @@
-ENV["OST_TIMEOUT"] = "1"
-
-require File.expand_path("test_helper", File.dirname(__FILE__))
-
-scope do
-  def ost(&job)
-    thread = Thread.new do
-      Ost[:events].each do |item|
-        begin
-          yield(item)
-        ensure
-          thread.kill
-        end
-      end
-    end
-
-    thread.join
-  end
-
-  def enqueue(id)
-    Ost[:events].push(id)
-  end
-
-  prepare do
-    Redis.current.flushall
-  end
-
-  setup do
-    Ost[:events].redis.quit
-    Redis.new
-  end
-
-  test "allows access to the queued items" do
-    enqueue(1)
-
-    assert_equal ["1"], Ost[:events].items
-  end
-
-  test "allows access to the underlying key" do
-    assert_equal 0, Ost[:events].key.llen
-  end
-
-  test "process items from the queue" do |redis|
-    enqueue(1)
-
-    results = []
-
-    ost do |item|
-      results << item
-    end
-
-    assert_equal [], Ost[:events].items
-    assert_equal ["1"], results
-  end
-
-  test "doesn't yield the block on timeout" do |redis|
-    results = []
-
-    Thread.new do
-      sleep 2
-      redis.lpush(Ost[:events].key, 1)
-    end
-
-    ost do |item|
-      results << item
-    end
-
-    assert_equal [], Ost[:events].items
-    assert_equal ["1"], results
-  end
-
-  test "halt processing a queue" do
-    Thread.new do
-      sleep 0.5
-      Ost[:always_empty].stop
-    end
-
-    Ost[:always_empty].each { }
-
-    assert true
-  end
-
-  test "halt processing all queues" do
-    Thread.new do
-      sleep 0.5
-      Ost.stop
-    end
-
-    t1 = Thread.new { Ost[:always_empty].each { } }
-    t2 = Thread.new { Ost[:always_empty_too].each { } }
-
-    t1.join
-    t2.join
-
-    assert true
-  end
-
-  test "maintains a backup queue for when worker dies" do
-    enqueue(1)
-
-    assert_equal 0, Ost[:events].backup.llen
-
-    begin
-      Ost[:events].each do |item|
-        item.some_error
-      end
-    rescue
-    end
-
-    assert_equal ["1"], Ost[:events].backup.lrange(0, -1)
-  end
-
-  test "cleans up the backup queue on success" do
-    enqueue(1)
-
-    done = false
-
-    Thread.new do
-      Ost[:events].each do |item|
-        assert_equal ["1"], Ost[:events].backup.lrange(0, -1)
-        done = true
-      end
-    end
-
-    until done; end
-
-    Ost[:events].stop
-
-    assert_equal 0, Ost[:events].backup.llen
-    assert_equal false, Ost[:events].backup.exists
-  end
-
-  test "uses same redis instance" do
-    queue = Ost['foo']
-
-    assert_equal queue.key.redis, queue.redis
-  end
-
-  test "report the queue size" do
-    queue = Ost[:size]
-
-    assert_equal 0, queue.size
-
-    queue << 1
-
-    assert_equal 1, queue.size
-
-    done = false
-
-    Thread.new do
-      Ost[:size].each do |item|
-        done = true
-      end
-    end
-
-    until done; end
-
-    assert_equal 0, queue.size
-  end
-end

data/test/test_helper.rb

@@ -1,6 +0,0 @@
-require "rubygems"
-require "cutest"
-
-$VERBOSE = 1
-
-require File.join(File.dirname(__FILE__), "..", "lib", "ost")