sparrow 0.3.1 → 0.4

data/Manifest.txt

@@ -5,7 +5,6 @@ Rakefile
 bin/sparrow
 lib/sparrow.rb
 lib/sparrow/queue.rb
-lib/sparrow/queues/disk.rb
 lib/sparrow/queues/memory.rb
 lib/sparrow/queues/sqlite.rb
 lib/sparrow/runner.rb

data/README.txt

@@ -11,6 +11,8 @@ Sparrow
   # The load Sparrow can cope with increases exponentially as you add to the cluster. 
   # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance.
   # 
+  # Sparrow is a in-memory queue but will persist the data to disk when receiving a term signal.
+  # 
   # Sparrow comes with built in support for daemonization and clustering. 
   # Also included are example libraries and clients. For example:
   # 

data/Rakefile

@@ -13,6 +13,7 @@ Hoe.new('sparrow', Sparrow::VERSION) do |p|
   p.url = 'http://code.google.com/p/sparrow'
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
   p.extra_deps << ['eventmachine', '>=0.10.0']
+  p.extra_deps << ['sqlite3-ruby', '>=1.2.2']
 end
 
 # vim: syntax=Ruby

data/bin/sparrow

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
-require File.join(File.dirname(__FILE__), '..', 'lib', 'sparrow')
+file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+require File.join(File.dirname(file), '..', 'lib', 'sparrow')
 Sparrow::Runner.run

data/lib/sparrow.rb

@@ -8,7 +8,7 @@ module Sparrow
   class SparrowError < StandardError #:nodoc:
   end
   
-  VERSION   = '0.3.1'
+  VERSION   = '0.4'
   
   @@options = {}
   
@@ -48,6 +48,5 @@ end
 require 'sparrow/server'
 require 'sparrow/queues/sqlite' rescue LoadError nil
 require 'sparrow/queues/memory'
-require 'sparrow/queues/disk'
 require 'sparrow/queue'
 require 'sparrow/runner'

data/lib/sparrow/queue.rb

@@ -7,12 +7,7 @@ module Sparrow
     
     class << self
       def get_queue(queue_name)
-        @@queues[queue_name] ||= case Sparrow.options[:type]
-          when 'memory': Sparrow::Queues::Memory.new(queue_name)
-          when 'sqlite': Sparrow::Queues::Sqlite.new(queue_name)
-          else
-            Sparrow::Queues::Disk.new(queue_name)
-          end
+        @@queues[queue_name] ||= Sparrow::Queues::Memory.new(queue_name)
       end
     
       def next_message(queue_name)
@@ -31,14 +26,17 @@ module Sparrow
       end
       
       def delete_all
+        @@queues.each {|name, q| q.clear! }
         @@queues = {}
-        FileUtils.rm_rf base_dir
-        FileUtils.mkdir_p base_dir
+      end
+      
+      def shutdown!
+        @@queues.each {|name, q| q.shutdown! }
       end
       
       def get_stats(queue_name)
         stats = {
-          :type =>              Sparrow.options[:type],
+          :type =>              'memory',
           :total_bytes =>       (File.size?(Sparrow.base_dir) || 0),
           :queues  =>           Dir.glob(File.join(Sparrow.base_dir, '*')).collect {|s| File.basename(s) }.join(','),
           :number_of_queues =>  queues.keys.length,
@@ -53,7 +51,6 @@ module Sparrow
         if queue_name
           queue = get_queue(queue_name)
           stats.merge!({
-            :bytes =>           Dir.glob(File.join(Sparrow.base_dir, queue_name + '**')).inject(0){|a, b| a += (File.size?(b) || 0); a },
             :total_items =>     queue.count_push, 
             :curr_items =>      queue.count
           })

data/lib/sparrow/queues/memory.rb

@@ -13,26 +13,51 @@ module Sparrow
         self.queue_data = []
         self.count_pop = 0
         self.count_push = 0
+        recover!
       end
       
       def pop
         self.count_pop += 1
-        queue_data.shift
+        self.queue_data.shift
       end
       
       def push(value)
         self.count_push += 1
-        queue_data.push(value)
+        self.queue_data.push(value)
       end
       
-      def clear
+      def clear!
         self.queue_data = []
+        self.sqlite.clear!
       end
       
       def count
         queue_data.length
       end
-         
+      
+      def to_disk!
+        copy = self.queue_data.dup
+        copy.each do |value|
+          self.sqlite.push(value)
+        end
+        self.queue_data = self.queue_data - copy
+      end
+      
+      def shutdown!
+        self.to_disk!
+      end
+      
+      def recover!
+        logger.debug "Recovering queue"
+        while msg = self.sqlite.pop
+          self.push(msg)
+        end
+      end
+      
+      def sqlite
+        @sqlite ||= Sparrow::Queues::Sqlite.new(self.queue_name)
+      end
+      
     end
   end
 end

data/lib/sparrow/queues/sqlite.rb

@@ -35,16 +35,17 @@ module Sparrow
     
       def pop
         id, value = db.get_first_row("SELECT * FROM queues LIMIT 1;")
+        return unless id and value
         db.execute("DELETE FROM queues WHERE id = ?", id)
         self.count_pop += 1
         value
       end
     
       def count
-        db.get_first_value("SELECT COUNT FROM queues")
+        db.get_first_value("SELECT count(*) FROM queues").to_i
       end
       
-      def clear
+      def clear!
         db.execute("DELETE FROM queues")
       end
       

data/lib/sparrow/runner.rb

@@ -22,7 +22,7 @@ module Sparrow
       
       self.options.merge!({
         :base_dir => base_dir,
-        :pid_dir => pid_dir,
+        :pid_dir  => pid_dir,
         :log_path => log_path
       })
       
@@ -64,6 +64,9 @@ module Sparrow
     end
     
     def stop
+      puts
+      puts "Writing queues to disk..."
+      Sparrow::Queue.shutdown!
       puts "Stopping Eventmachine Server"
       EventMachine::stop
     end
@@ -72,7 +75,7 @@ module Sparrow
       OptionParser.new do |opts|
         opts.summary_width = 25
         opts.banner = "Sparrow (#{VERSION})\n\n",
-                      "Usage: sparrow [-b path] [-t type] [-h host] [-p port] [-P file]\n",
+                      "Usage: sparrow [-b path] [-h host] [-p port] [-P file]\n",
                       "               [-d] [-k port] [-l file] [-e]\n",
                       "       sparrow --help\n",
                       "       sparrow --version\n"
@@ -84,10 +87,6 @@ module Sparrow
           options[:base_dir] = File.expand_path(v)
         end
         
-        opts.on("-t", "--type QUEUE_TYPE", String, "Type of queue (disk/memory/sqlite).", "(default: #{options[:type]})") do |v|
-          options[:type] = v
-        end
-        
         opts.separator ""; opts.separator "Network:"
         
         opts.on("-h", "--host HOST", String, "Specify host", "(default: #{options[:host]})") do |v|
@@ -154,7 +153,7 @@ module Sparrow
         puts f
         pid = IO.read(f).chomp.to_i
         FileUtils.rm f
-        Process.kill(9, pid)
+        Process.kill(15, pid) # TERM
         puts "killed PID: #{pid}"
         rescue => e
           puts "Failed to kill! #{k}: #{e}"

metadata

@@ -1,33 +1,57 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4
-specification_version: 1
 name: sparrow
 version: !ruby/object:Gem::Version 
-  version: 0.3.1
-date: 2008-02-03 00:00:00 +00:00
-summary: Simple file based messagine queue using the memcache protocol
-require_paths: 
-- lib
-email: info@eribium.org
-homepage: http://code.google.com/p/sparrow
-rubyforge_project: Sparrow
-description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached.  # That means you can use Sparrow with any memcached client library (Ruby or otherwise).  #  # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second.  # The load Sparrow can cope with increases exponentially as you add to the cluster.  # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. #  # Sparrow comes with built in support for daemonization and clustering.  # Also included are example libraries and clients. For example: #  # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name']       #=> 1 Pull next msg from queue # m['queue_name']       #=> nil # m.delete('queue_name) # Delete queue #  # # or using the included client: #  # class MyQueue < MQ3::Queue #   def on_message #     logger.info \"Received msg with args: #{args.inspect}\" #   end # end #  # MyQueue.servers = [ #   MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run #  # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't  # be the same order when they're removed. #  # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. #  # Call sparrow with --help for usage options #  # The daemonization won't work on Windows.  #  # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow #  # Sparrow was inspired by Twitter's Starling"
-autorequire: 
-default_executable: 
-bindir: bin
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
+  version: "0.4"
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
 - Alex MacCAw
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-08-02 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.10.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.7.0
+    version: 
+description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached.  # That means you can use Sparrow with any memcached client library (Ruby or otherwise).  #  # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second.  # The load Sparrow can cope with increases exponentially as you add to the cluster.  # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. #  # Sparrow is a in-memory queue but will persist the data to disk when receiving a term signal. #  # Sparrow comes with built in support for daemonization and clustering.  # Also included are example libraries and clients. For example: #  # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name']       #=> 1 Pull next msg from queue # m['queue_name']       #=> nil # m.delete('queue_name) # Delete queue #  # # or using the included client: #  # class MyQueue < MQ3::Queue #   def on_message #     logger.info \"Received msg with args: #{args.inspect}\" #   end # end #  # MyQueue.servers = [ #   MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run #  # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't  # be the same order when they're removed. #  # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. #  # Call sparrow with --help for usage options #  # The daemonization won't work on Windows.  #  # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow #  # Sparrow was inspired by Twitter's Starling"
+email: info@eribium.org
+executables: 
+- sparrow
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
 files: 
 - History.txt
 - Manifest.txt
@@ -36,43 +60,37 @@ files:
 - bin/sparrow
 - lib/sparrow.rb
 - lib/sparrow/queue.rb
-- lib/sparrow/queues/disk.rb
 - lib/sparrow/queues/memory.rb
 - lib/sparrow/queues/sqlite.rb
 - lib/sparrow/runner.rb
 - lib/sparrow/server.rb
 - lib/sparrow/utils.rb
-test_files: []
-
+has_rdoc: true
+homepage: http://code.google.com/p/sparrow
+post_install_message: 
 rdoc_options: 
 - --main
 - README.txt
-extra_rdoc_files: 
-- History.txt
-- Manifest.txt
-- README.txt
-executables: 
-- sparrow
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: eventmachine
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.10.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.5.0
-    version: 
+rubyforge_project: Sparrow
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Simple file based messagine queue using the memcache protocol
+test_files: []
+

data/lib/sparrow/queues/disk.rb

@@ -1,109 +0,0 @@
-require 'fileutils'
-module Sparrow
-  module Queues
-    class Disk
-      include Sparrow::Miscel
-      
-      TRX_CMD_PUSH = "\000".freeze
-      TRX_CMD_POP = "\001".freeze
-
-      TRX_PUSH = "\000%s%s".freeze
-      TRX_POP = "\001".freeze
-    
-      attr_accessor :queue_name
-      attr_accessor :trxr
-      attr_accessor :trxw
-      attr_accessor :count_pop
-      attr_accessor :count_push
-    
-      def initialize(queue_name)
-        self.queue_name = queue_name
-        self.count_pop = 0
-        self.count_push = 0
-        open_queue
-      end
-    
-      def push(value)
-        value = value.to_s
-        size = [value.size].pack("I")
-        data = sprintf(TRX_PUSH, size, value)
-        trxw.seek(0, IO::SEEK_END)
-        trxw.write data
-        # trxw.fsync
-        rotate_queue if trxw.pos > max_log_size
-        self.count_push += 1
-        value
-      end
-    
-      def pop        
-        while !trxr.eof?
-          s_pos = trxr.pos
-          cmd = trxr.read(1)
-          if cmd != TRX_CMD_POP and cmd != TRX_CMD_PUSH
-            logger.fatal 'Corrupt queue'
-            return
-          end
-          raw_size = trxr.read(4)
-          size = raw_size.unpack("I").first
-          value = trxr.read(size)
-          next if cmd == TRX_CMD_POP
-          e_pos = trxr.pos
-          trxr.seek(s_pos, IO::SEEK_SET)
-          trxr.write(TRX_POP)
-          # trxr.fsync
-          trxr.pos = e_pos
-          next unless value
-          self.count_pop += 1
-          return value
-        end
-        
-        if trxr.path == queue_path
-          File.truncate(trxr.path, 0)
-        else
-          FileUtils.rm_rf trxr.path
-        end
-        open_reader
-        nil
-      end
-      
-      def clear
-        dirs = Dir.glob(queue_path) | Dir.glob(queue_path + '.*')
-        FileUtils.rm_rf(dirs) unless dirs.empty?
-      end
-      
-      def count
-        self.count_push - self.count_pop
-      end
-      
-      private
-      
-      def queue_path
-        File.join(base_dir, queue_name)
-      end
-      
-      def rotate_queue
-        File.rename(queue_path, File.join(base_dir, "#{queue_name}.#{Time.now.to_i}"))
-        open_writer
-      end
-      
-      def open_writer
-        self.trxw = File.open(queue_path, 'a+')
-      end
-      
-      def open_reader    
-        old_queue = Dir.glob(queue_path + '.*').first
-        self.trxr = File.open(old_queue||queue_path, 'r+')
-      end
-    
-      def open_queue
-        open_writer
-        open_reader
-      end
-      
-      def max_log_size
-        @max_log_size ||= (options[:log_size] || 16) * (1024**2) # 16mb
-      end
-
-    end
-  end
-end