riaq 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZWI0ODkwNGRhMWFjNjRhNzM3OGJiMTRjOWNjZGZlM2FhYTY2NTVkYg==
+  data.tar.gz: !binary |-
+    OWQ2MjZiYTcwZWEyMjk2OGY3NmM2NjlhZGNmMTU1M2ZhZTYzZmUyNg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    M2YwNjgwMDc3NDM5NWUxOGNhZWQ3ODU3NGUwMWE5OTFkZTQyZGFkMmRkYTNl
+    NmExMDM4NmI3YTJiMzE0OWU1OGZmYzZiYThlOTU4YmZmYmZiZjg5MjA3YjAx
+    ZGZhNjYyYjgxYTNjNmQxNzZkNmRjNTc4YjdjOGZkNTkxM2U5Zjc=
+  data.tar.gz: !binary |-
+    MGE3YWY4OGQyMGEyNzg4MTYzYTc3OWZkNGQwNWIxNjEzYTA5ZmQ2NmQ4MGQx
+    N2E2MTg4YjA2OGM5OGUwZmIwY2VmODJhZGYzMTE2NGJiNjFmODM5YzNmOTdl
+    YjJkMGMzZWNjMmY4YjVlODI5M2MzN2MwNjNkYjk0MTFjYWM0YmM=

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Matías Flores
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+# Riaq
+
+Experimental attempt of providing Riak based queues and workers,
+heavily inspired by [Ost](https://github.com/soveran/ost).
+
+Due to Riak's distributed capabilities and lack of atomic operations,
+as soon as you launch more than one worker there's no guarantee that
+each item added to the queue will be processed just once. If you can't
+live with this limitation, then this tool is not for you.
+
+## Description
+
+*Riaq* makes it easy to enqueue object ids and process them with
+workers.
+
+Say you want to process video uploads. In your application you will
+have something like this:
+
+```ruby
+Riaq[:videos_to_process].push(@video.id)
+```
+
+Then, you will have a worker that will look like this:
+
+```ruby
+require "riaq"
+
+Riaq[:videos_to_process].each do |id|
+  # Do something with it!
+end
+```
+
+## Usage
+
+*Riaq* connects to Riak automatically with the default options.
+
+You can customize the connection by calling `connect`.
+
+Then you only need to refer to a queue for it to pop into existence:
+
+```ruby
+Riaq[:rss_feeds] << @feed.id
+```
+
+A worker is a Ruby file with this basic code:
+
+```ruby
+require "riaq"
+
+Riaq[:rss_feeds].each do |id|
+  # ...
+end
+```
+
+It will pop items from the queue as soon as they become available.
+
+Note that in these examples we are pushing numbers to the queue. As
+we have unlimited queues, each queue should be specialized and the
+workers must be smart enough to know what to do with the numbers they
+pop.
+
+### Available methods
+
+`Riaq.connect`: configure the connection to Riak. It accepts
+the same options as [the Riak Ruby client](https://github.com/basho/riak-ruby-client).
+
+`Riaq.stop`: halt processing for all queues.
+
+`Riaq[:example].push item`, `Riaq[:some_queue] << item`: add `item` to
+the `:example` queue.
+
+`Riaq[:example].each { |item| ... }`: consume `item` from the `:example` queue.
+
+`Riaq[:example].stop`: halt processing for the `example` queue.
+
+## Priorities
+
+There's no concept of priorities, as each queue is specialized and you
+can create as many as you want. For example, nothing prevents the
+creation of the `:example_high_priority` or the
+`:example_low_priority` queues.
+
+## Installation
+
+    $ gem install riaq
+
+## License
+
+Distributed under the terms of the MIT license.
+See bundled [LICENSE](https://github.com/matflores/riaq/blob/master/LICENSE)
+file for more info.

data/Rakefile

@@ -0,0 +1,5 @@
+task :test do
+  Dir["test/*.rb"].each { |file| load file }
+end
+
+task :default => :test

data/lib/riaq.rb

@@ -0,0 +1,92 @@
+require "riak"
+
+module Riaq
+  PENDING = 0
+  PROCESSING = 1
+
+  class Queue
+    attr :bucket
+
+    def initialize(name)
+      @bucket = riak.bucket("riaq:#{name}")
+    end
+
+    def push(value)
+      item = @bucket.new(Time.now.to_f)
+      item.content_type = "text/plain"
+      item.data = value
+      item.indexes = { status_int: PENDING }
+      item.store
+    end
+
+    def each(&block)
+      @stopping = false
+
+      loop do
+        break if @stopping
+
+        key = bucket.get_index("status_int", PENDING, max_results: 1).first
+
+        next unless key
+
+        item = @bucket.get(key)
+        item.indexes = { status_int: PROCESSING }
+        item.store
+
+        block.call(item.data)
+
+        item.delete
+      end
+    end
+
+    def flush
+      bucket.get_index("$bucket", @bucket.name).each do |key|
+        @bucket.delete(key)
+      end
+    end
+
+    def stop
+      @stopping = true
+    end
+
+    def items
+      Riak::MapReduce.new(riak).index(@bucket.name, "status_int", PENDING).map("Riak.mapValues", keep: true).run
+    end
+
+    def size
+      bucket.get_index("status_int", PENDING).size
+    end
+
+    def empty?
+      size == 0
+    end
+
+    alias << push
+
+    def riak
+      @riak ||= Riak::Client.new(Riaq.options)
+    end
+  end
+
+  @queues = Hash.new do |hash, key|
+    hash[key] = Queue.new(key)
+  end
+
+  def self.[](queue)
+    @queues[queue]
+  end
+
+  def self.stop
+    @queues.each { |_, queue| queue.stop }
+  end
+
+  @options = nil
+
+  def self.connect(options = {})
+    @options = options
+  end
+
+  def self.options
+    @options || {}
+  end
+end

data/riaq.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name        = "riaq"
+  s.version     = "0.0.1"
+  s.summary     = "Riak based queues and workers, inspired by Ost."
+  s.description = "Riaq lets you manage queues and workers with Riak."
+  s.authors     = ["Matías Flores"]
+  s.email       = ["flores.matias@gmail.com"]
+  s.homepage    = "http://github.com/soveran/ost"
+  s.license     = "MIT"
+
+  s.files = Dir[
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "lib/**/*.rb",
+    "*.gemspec",
+    "test/*.*"
+  ]
+
+  s.add_dependency "riak-client", "~> 1.4.3"
+  s.add_development_dependency "protest", "~> 0.5.1"
+end

data/test/riaq_test.rb

@@ -0,0 +1,85 @@
+require "protest"
+require_relative "../lib/riaq"
+
+Protest.context "Riaq" do
+  setup do
+    Riaq[:test].flush
+  end
+
+  test "access queued items" do
+    push "1"
+
+    assert_equal ["1"], Riaq[:test].items
+  end
+
+  test "query the number of queued items" do
+    push "1"
+
+    assert_equal 1, Riaq[:test].size
+  end
+
+  test "check queue emptiness" do
+    assert Riaq[:test].empty?
+
+    push "1"
+
+    assert !Riaq[:test].empty?
+  end
+
+  test "process items from the queue in order" do |redis|
+    %w(1 2 3).each { |i| push i }
+
+    results = []
+
+    process do |item|
+      results << item
+    end
+
+    assert Riaq[:test].empty?
+    assert_equal ["1", "2", "3"], results
+  end
+
+  test "halt processing a queue" do
+    Thread.new do
+      sleep 0.5
+      Riaq[:always_empty].stop
+    end
+
+    Riaq[:always_empty].each { }
+
+    assert true
+  end
+
+  test "halt processing all queues" do
+    Thread.new do
+      sleep 0.5
+      Riaq.stop
+    end
+
+    t1 = Thread.new { Riaq[:always_empty].each { } }
+    t2 = Thread.new { Riaq[:always_empty_too].each { } }
+
+    t1.join
+    t2.join
+
+    assert true
+  end
+
+  def push(id)
+    Riaq[:test].push(id)
+  end
+
+  def process(&job)
+    thread = Thread.new do
+      Riaq[:test].each do |item|
+        begin
+          yield(item)
+        ensure
+          thread.kill if Riaq[:test].empty?
+        end
+      end
+    end
+
+    thread.join
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: riaq
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Matías Flores
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: riak-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.3
+- !ruby/object:Gem::Dependency
+  name: protest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+description: Riaq lets you manage queues and workers with Riak.
+email:
+- flores.matias@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- lib/riaq.rb
+- riaq.gemspec
+- test/riaq_test.rb
+homepage: http://github.com/soveran/ost
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Riak based queues and workers, inspired by Ost.
+test_files: []
+has_rdoc: