karait 0.0.1

data/.gitignore

@@ -0,0 +1,13 @@
+html
+pkg
+doc
+*.gem
+nbproject
+*.bundle
+*.o
+*~
+*#*
+*.class
+*.swp
+*.pid
+*.log

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+
+gem "mongo", "~> 1.3.1"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,24 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    bson (1.3.1)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    mongo (1.3.1)
+      bson (>= 1.3.1)
+    rake (0.9.2)
+    rcov (0.9.10)
+    shoulda (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  mongo (~> 1.3.1)
+  rcov
+  shoulda

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 bcoe
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,74 @@
+Karait
+======
+
+A ridiculously simple queuing system, with clients in various languages, built on top of MongoDB.
+
+Contributing to karait
+----------------------
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+Usage
+-----
+
+_Writing to a queue_
+
+```ruby
+require 'karait'
+
+queue = Karait::Queue.new(
+    :host => 'localhost', # MongoDB host. Defaults to localhost.
+    :port => 27017, # MongoDB port. Defaults to 27017.
+    :database => 'karait', # Database that will store the karait queue. Defaults to karait.
+    :queue => 'messages', # The capped collection that karait writes to. Defaults to messages.
+    :average_message_size => 8192, # How big do you expect the messages will be in bytes? Defaults to 8192.
+    :queue_size => 4096 # How many messages should be allowed in the queue. Defaults to 4096.
+)
+
+queue.write({
+	:name => 'Benjamin',
+	:action => 'Rock'
+})
+
+# or
+
+message = Karait::Message.new
+message.name = 'Benjamin'
+message.action = 'Rock!'
+
+queue.write(message, :routing_key => 'my_routing_key', :expire => 3.0)
+```
+
+_Reading from a queue_
+
+```ruby
+require 'karait'
+
+queue = Karait::Queue.new
+
+message = queue.read().first
+print "#{message.name}"
+
+message.delete
+
+# or
+
+message = queue.read(:routing_key => 'my_routing_key').first
+print "#{message.action}"
+
+message.delete
+```
+
+See unit tests for more documentation.
+
+Copyright
+---------
+
+Copyright (c) 2011 Attachments.me. See LICENSE.txt for
+further details.

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "karait"
+  gem.homepage = "http://github.com/bcoe/karait"
+  gem.license = "MIT"
+  gem.summary = %Q{A ridiculously simple cross-language queuing system, built on top of MongoDB.}
+  gem.description = %Q{A ridiculously simple cross-language queuing system, built on top of MongoDB.}
+  gem.email = "bencoe@gmail.com"
+  gem.authors = ["bcoe"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "karait #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/karait.rb

@@ -0,0 +1,5 @@
+require 'message'
+require 'queue'
+
+module Karait
+end

data/lib/message.rb

@@ -0,0 +1,85 @@
+module Karait
+  class Message
+    
+    include Karait
+    
+    ASSIGN_VARIABLE_REGEX = /^([a-z_][a-zA-Z_0-9]*)=$/
+    VARIABLE_REGEX = /^([a-z_][a-zA-Z_0-9]*)$/
+    BLACKLIST = {
+        '_meta' => true,
+        '_id' => true,
+        '_expired' => true
+    }
+    
+    def initialize(raw_message={}, queue_collection=nil)
+      @source = raw_message
+      @queue_collection = queue_collection
+      @variables_to_serialize = {}
+      set_expired
+      add_accessors raw_message
+    end
+    
+    def to_hash
+      return @variables_to_serialize
+    end
+    
+    def delete
+      @queue_collection.update(
+          {
+              '_id' => @source['_id']
+          },
+          {
+              '$set' => {
+                  '_meta.expired' => true
+              }
+          }
+      )
+    end
+    
+    def expired?
+      return @expired
+    end
+    
+    def get(key)
+      return @variables_to_serialize[key.to_s]
+    end
+    
+    def _get_id
+      return @source['_id']
+    end
+    
+    private
+    
+    def set_expired
+      @expired = false
+      
+      current_time = Time.now().to_f
+      meta = @source.fetch('_meta', {})
+      
+      return if meta.fetch('expire', -1.0) == -1.0
+      
+      if current_time - meta.fetch('timestamp', 0.0) > meta.fetch('expire', -1.0):
+        @expired = true
+      end
+    end
+    
+    def add_accessors(hash)
+      hash.each do |k, v|
+        if not Message::BLACKLIST.has_key? k
+          @variables_to_serialize[k.to_s] = v
+        end
+      end
+    end
+    
+    def method_missing(sym, *args, &block)
+      if matches = sym.to_s.match(Message::ASSIGN_VARIABLE_REGEX) and args.count == 1
+        @variables_to_serialize[matches[1]] = args[0]
+      elsif matches = sym.to_s.match(Message::VARIABLE_REGEX) and args.count == 0
+        return @variables_to_serialize[matches[1]]
+      else
+        super(sym, *args, &block)
+      end
+    end
+    
+  end
+end

data/lib/queue.rb

@@ -0,0 +1,123 @@
+require 'mongo'
+
+module Karait
+  class Queue
+    
+    include Karait
+    
+    MESSAGES_READ = 10
+    
+    def initialize(opts={})
+      set_instance_variables opts
+      create_mongo_connection
+    end
+    
+    def write(message, opts={})
+      if message.class == Hash
+        message_dict = message
+      else
+        message_dict = message.to_hash
+      end
+      
+      message_dict[:_meta] = {
+        :expire => opts.fetch(:expire, -1.0),
+        :timestamp => Time.now().to_f,
+        :expired => false
+      }
+      
+      message_dict[:_meta][:routing_key] = opts.fetch(:routing_key) if opts[:routing_key]
+      
+      @queue_collection.insert(message_dict, :safe => true)
+    end
+    
+    def read(opts={})
+      messages = []
+      
+      conditions = {
+          '_meta.expired' => false
+      }
+      
+      if opts[:routing_key]
+        conditions['_meta.routing_key'] = opts[:routing_key]
+      else
+        conditions['_meta.routing_key'] = {
+          '$exists' => false
+        }
+      end
+      
+      @queue_collection.find(conditions).limit(opts.fetch(:messages_read, Queue::MESSAGES_READ)).each do |raw_message|
+        message = Karait::Message.new(raw_message=raw_message, queue_collection=@queue_collection)
+        if message.expired?
+          message.delete()
+        else
+          messages << message
+        end
+      end
+      
+      return messages
+    end
+    
+    def delete_messages(messages)
+      ids = []
+      messages.each {|message| ids << message._get_id}
+      @queue_collection.update(
+          {
+              '_id' => {
+                '$in' => ids
+              }
+          },
+          {
+              '$set' => {
+                  '_meta.expired' => true
+              }
+          },
+          :multi => true,
+          :safe => true
+      )
+    end
+    
+    private
+    
+    def set_instance_variables(opts)
+      
+      defaults = {
+        :host => 'localhost',
+        :port => 27017,
+        :database => 'karait',
+        :queue => 'messages',
+        :average_message_size => 8192,
+        :queue_size => 4096
+      }.merge(opts)
+      
+      @host = defaults[:host]
+      @port = defaults[:port]
+      @database = defaults[:database]
+      @queue = defaults[:queue]
+      @average_message_size = defaults[:average_message_size]
+      @queue_size = defaults[:queue_size]
+      
+    end
+    
+    def create_mongo_connection
+      @connection = Mongo::Connection.new(
+        @host,
+        @port
+      )
+      @database = @connection[@database]
+      create_capped_collection
+      @queue_collection = @database[@queue]
+      @queue_collection.create_index('_id')
+      @queue_collection.create_index('_meta.routing_key')
+    end
+    
+    def create_capped_collection
+      @database.create_collection(
+        @queue,
+        :size => (@average_message_size * @queue_size),
+        :capped => true,
+        :max => @queue_size
+      )
+    end
+    
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'karait'
+
+class Test::Unit::TestCase
+end

data/test/test_message.rb

@@ -0,0 +1,124 @@
+require 'helper'
+require 'karait'
+require 'yaml'
+
+class TestMessage < Test::Unit::TestCase
+  
+  def setup
+    Mongo::Connection.new()['karait_test']['queue_test'].drop()
+  end
+  
+  should "should add attribute accessors when initialized with a raw message" do
+    message = Karait::Message.new(
+      raw_message={
+        :apple => 5,
+        :banana => 3
+      }
+    )
+    
+    message.apple = 6
+    assert_equal 6, message.apple
+    assert_equal 3, message.banana
+  end
+  
+  should "add new attribute accessors for missing methods" do
+    message = Karait::Message.new
+    message.foo = 9
+    assert_equal 9, message.foo
+  end
+  
+  should "return a hash of appropriate instance variables when to_hash is called" do
+    message = Karait::Message.new(
+      raw_message={
+        :apple => 5,
+        :banana => 3
+      }
+    )
+    message.apple = 6
+    message.foo = {'bar' => 9}
+    message.bar = [27]
+    
+    hash = message.to_hash
+    
+    assert_equal 6, hash['apple']
+    assert_equal 3, hash['banana']
+    assert_equal 9, hash['foo']['bar']
+    assert_equal 27, hash['bar'][0]
+    assert_equal 4, hash.keys.count
+  end
+  
+  should "not copy blacklisted keys when to_hash called" do
+      raw_message = {
+          '_id' => 'foobar',
+          '_meta' => {
+              'foo' => 2
+          }
+      }
+      message = Karait::Message.new(raw_message=raw_message)
+      hash = message.to_hash
+      
+      assert_equal 0, hash.keys.count
+  end
+  
+  should "set expired to true when delete is called on a message" do
+      collection = Mongo::Connection.new()['karait_test']['queue_test']
+      collection.insert({
+          'routing_key' => 'foobar',
+          'apple' => 3,
+          'banana' => 5,
+          '_meta' => {
+              'timestamp' => 2523939,
+              'expire' => 20393,
+              'expired' => false
+          }
+      })
+      raw_message = collection.find_one({'_meta.expired' => false})
+      assert_equal 3, raw_message['apple']
+      message = Karait::Message.new(raw_message=raw_message, queue_collection=collection)
+      message.delete()
+      assert_equal 0, collection.find({'_meta.expired' => false}).count
+  end
+
+  should "set expired to true if current time minus timestamp is greater than expire" do
+    raw_message = {
+        'routing_key' => 'foobar',
+        'apple' => 3,
+        'banana' => 5,
+        '_meta' => {
+            'timestamp' => 0,
+            'expire' => 10,
+            'expired' => false
+        }
+    }
+
+    message = Karait::Message.new(raw_message=raw_message)
+    assert_equal true, message.expired?
+  end
+  
+  should "not set expired to true if expire is -1.0" do
+    raw_message = {
+        'routing_key' => 'foobar',
+        'apple' => 3,
+        'banana' => 5,
+        '_meta' => {
+            'timestamp' => 0,
+            'expire' => -1.0,
+            'expired' => false
+        }
+    }
+    
+    message = Karait::Message.new(raw_message=raw_message)
+    assert_equal false, message.expired?
+  end
+  
+  should "allow a get method to be called to retrieve keys that conflict with class variables, e.g., send" do
+    message = Karait::Message.new(
+      raw_message={
+        :send => 5,
+        :banana => 3
+      }
+    )
+    assert_equal 3, message.get(:banana)
+    assert_equal 5, message.get(:send)
+  end
+end

data/test/test_queue.rb

@@ -0,0 +1,214 @@
+require 'helper'
+require 'mongo'
+require 'karait'
+
+class TestQueue < Test::Unit::TestCase
+  
+  def setup
+    Mongo::Connection.new()['karait_test']['queue_test'].drop()
+  end
+  
+  should "initialize a capped collection when a queue is created" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test',
+      :average_message_size => 8192,
+      :queue_size => 4096
+    )
+    
+    options = Mongo::Connection.new()['karait_test']['queue_test'].options
+    
+    assert_equal true, options['capped']
+    assert_equal 4096, options['max']
+    assert_equal (8192 * 4096), options['size']
+  end
+  
+  should "attach to a mongo queue collection that already exists" do
+    collection = Mongo::Connection.new()['karait_test']['queue_test']
+    collection.insert({
+      :message => {
+        :apple => 3,
+        :banana => 5
+      },
+      :_meta => {
+        :timestamp => 2523939,
+        :expire => 20393,
+        :routing_key => 'foo_key'
+      }
+    })
+    assert_equal 1, collection.count()
+    
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+
+    collection = Mongo::Connection.new()['karait_test']['queue_test']
+    assert_equal 1, collection.count()
+    assert_equal nil, collection.options()
+  end
+  
+  should "write a dictionary into the mongo queue collection" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test',
+      :average_message_size => 8192,
+      :queue_size => 4096
+    )
+    
+    queue.write({
+      :apple => 5,
+      :banana => 6,
+      :inner_object => {
+        :foo => 1,
+        :bar => 2
+      }
+    })
+
+    collection = Mongo::Connection.new()['karait_test']['queue_test']
+    obj = collection.find_one()
+    assert_equal 6, obj['banana']
+    assert_equal 2, obj['inner_object']['bar']
+    assert obj['_meta']['expire']
+    assert obj['_meta']['timestamp']
+  end
+  
+  should "write a message object into the mongo queue collection" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    message = Karait::Message.new
+    message.apple = 5
+    message.banana = {
+      :foo => 2
+    }
+    queue.write message
+    
+    collection = Mongo::Connection.new()['karait_test']['queue_test']
+    obj = collection.find_one
+    assert_equal 5, obj['apple']
+    assert_equal 2, obj['banana']['foo']
+  end
+
+  should "read a message object from the mongo queue collection" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+  
+    write_message = Karait::Message.new
+    write_message.apple = 5
+    write_message.banana = 6
+    write_message.inner_object = {
+      'foo' => 1,
+      'bar' => 2
+    }
+    queue.write write_message
+  
+    read_message = queue.read()[0]
+    assert_equal 5, read_message.apple
+    assert_equal 2, read_message.inner_object['bar']
+    assert_equal 3, read_message.to_hash.keys.count
+  end
+  
+  should "return messages in lifo order" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write Karait::Message.new({'foo' => 1})
+    queue.write Karait::Message.new({:foo => 2})
+    queue.write Karait::Message.new({'foo' => 3})
+    messages = queue.read()
+    
+    assert_equal 1, messages[0].foo
+    assert_equal 2, messages[1].foo
+    assert_equal 3, messages[2].foo
+  end
+  
+  should "only return messages with the appropriate routing key when it's provided" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write(Karait::Message.new({:foo => 1}), :routing_key => 'foobar')
+    queue.write Karait::Message.new({:foo => 2})
+
+    messages = queue.read(:routing_key => 'foobar')
+    assert_equal 1, messages.count
+    assert_equal 1, messages[0].foo
+  end
+  
+  should "only return messages with no routing key when none is provided" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write(Karait::Message.new({:foo => 1}), :routing_key => 'foobar')
+    queue.write Karait::Message.new({:foo => 2})
+    
+    messages = queue.read()
+    assert_equal 1, messages.count
+    assert_equal 2, messages[0].foo
+  end
+  
+  should "should no longer return a message when delete is called on it" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write Karait::Message.new({:foo => 1})
+    messages = queue.read()
+    assert_equal 1, messages.count
+    assert_equal 1, messages[0].foo
+    
+    messages[0].delete
+    messages = queue.read()
+    assert_equal 0, messages.count
+  end
+
+  should "not remove message immediately with expire set" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write(Karait::Message.new({:foo => 1}), :expire => 0.5)
+    sleep(0.1)
+    messages = queue.read()
+    assert_equal 1, messages.count
+  end
+  
+  should "remove message once expire time is passed" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write(Karait::Message.new({:foo => 1}), :expire => 0.1)
+    sleep(0.2)
+    messages = queue.read()
+    assert_equal 0, messages.count
+  end
+  
+  
+  should "remove all messages in array when delete messages called" do
+    queue = Karait::Queue.new(
+      :database => 'karait_test',
+      :queue => 'queue_test'
+    )
+    
+    queue.write Karait::Message.new({'foo' => 1})
+    queue.write Karait::Message.new({:foo => 2})
+    queue.write Karait::Message.new({'foo' => 3})
+    messages = queue.read()
+    queue.delete_messages messages
+    messages = queue.read()
+    assert_equal 0, messages.count
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification 
+name: karait
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- bcoe
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-31 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: A ridiculously simple cross-language queuing system, built on top of MongoDB.
+email: bencoe@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- VERSION
+- lib/karait.rb
+- lib/message.rb
+- lib/queue.rb
+- test/helper.rb
+- test/test_message.rb
+- test/test_queue.rb
+has_rdoc: true
+homepage: http://github.com/bcoe/karait
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: A ridiculously simple cross-language queuing system, built on top of MongoDB.
+test_files: []
+