sparrow 0.2 → 0.3

data/lib/sparrow.rb

@@ -8,7 +8,7 @@ module Sparrow
   class SparrowError < StandardError #:nodoc:
   end
   
-  VERSION   = '0.2'
+  VERSION   = '0.3'
   
   @@options = {}
   
@@ -32,15 +32,15 @@ module Sparrow
     end
   
     def base_dir
-      options[:base_dir] || File.join(%w( / var spool sparrow ))
+      options[:base_dir] || File.join(%w( / var spool sparrow base ))
     end
   
     def log_path
       options[:log_path] || File.join(%w( / var run sparrow.log ))
     end
   
-    def pid_path
-      options[:pid_path] || File.join(%w( / var run sparrow.#{options[:port]}.pid ))
+    def pid_dir
+      options[:pid_dir] || File.join(%w( / var run sparrow pids ))
     end
   end
 end

data/lib/sparrow/queue.rb

@@ -35,6 +35,31 @@ module Sparrow
         FileUtils.rm_rf base_dir
         FileUtils.mkdir_p base_dir
       end
+      
+      def get_stats(queue_name)
+        stats = {
+          :type =>              Sparrow.options[:type],
+          :total_bytes =>       (File.size?(Sparrow.base_dir) || 0),
+          :queues  =>           Dir.glob(File.join(Sparrow.base_dir, '*')).collect {|s| File.basename(s) }.join(','),
+          :number_of_queues =>  queues.keys.length,
+          :debug =>             Sparrow.options[:debug],
+          :pid =>               Process.pid,
+          :uptime =>            Time.now - Sparrow.options[:start_time],
+          :time =>              Time.now.to_i,
+          :version =>           Sparrow::VERSION,
+          :rusage_user =>       Process.times.utime,
+          :rusage_system =>     Process.times.stime
+        }
+        if queue_name
+          queue = get_queue(queue_name)
+          stats.merge!({
+            :bytes =>           Dir.glob(File.join(Sparrow.base_dir, queue_name + '**')).inject(0){|a, b| a += (File.size?(b) || 0); a },
+            :total_items =>     queue.count_push, 
+            :curr_items =>      queue.count
+          })
+        end
+        stats
+      end  
     end
     
   end

data/lib/sparrow/queues/disk.rb

@@ -13,11 +13,13 @@ module Sparrow
       attr_accessor :queue_name
       attr_accessor :trxr
       attr_accessor :trxw
-      attr_accessor :count_all
+      attr_accessor :count_pop
+      attr_accessor :count_push
     
       def initialize(queue_name)
         self.queue_name = queue_name
-        self.count_all = 0
+        self.count_pop = 0
+        self.count_push = 0
         open_queue
       end
     
@@ -27,9 +29,9 @@ module Sparrow
         data = sprintf(TRX_PUSH, size, value)
         trxw.seek(0, IO::SEEK_END)
         trxw.write data
-        trxw.fsync
+        # trxw.fsync
         rotate_queue if trxw.pos > max_log_size
-        self.count_all += 1
+        self.count_push += 1
         value
       end
     
@@ -48,9 +50,10 @@ module Sparrow
           e_pos = trxr.pos
           trxr.seek(s_pos, IO::SEEK_SET)
           trxr.write(TRX_POP)
-          trxr.fsync
+          # trxr.fsync
           trxr.pos = e_pos
           next unless value
+          self.count_pop += 1
           return value
         end
         
@@ -69,6 +72,7 @@ module Sparrow
       end
       
       def count
+        self.count_push - self.count_pop
       end
       
       private

data/lib/sparrow/queues/memory.rb

@@ -5,20 +5,23 @@ module Sparrow
       
       attr_accessor :queue_name
       attr_accessor :queue_data
-      attr_accessor :count_all
+      attr_accessor :count_pop
+      attr_accessor :count_push
       
       def initialize(queue_name)
         self.queue_name = queue_name
         self.queue_data = []
-        self.count_all = 0
+        self.count_pop = 0
+        self.count_push = 0
       end
       
       def pop
+        self.count_pop += 1
         queue_data.shift
       end
       
       def push(value)
-        self.count_all += 1
+        self.count_push += 1
         queue_data.push(value)
       end
       

data/lib/sparrow/queues/sqlite.rb

@@ -6,11 +6,13 @@ module Sparrow
     
       attr_accessor :queue_name
       attr_accessor :db
-      attr_accessor :count_all
+      attr_accessor :count_pop
+      attr_accessor :count_push
     
       def initialize(queue_name)
         self.queue_name = queue_name
-        self.count_all = 0
+        self.count_pop = 0
+        self.count_push = 0
         db_exists = File.exists?(db_path)
         self.db = SQLite3::Database.new( db_path )
         if !db_exists
@@ -26,14 +28,15 @@ module Sparrow
       end
         
       def push(value)
-        self.count_all += 1
         db.execute("INSERT INTO queues (data) VALUES (?);", value)
+        self.count_push += 1
         value
       end
     
       def pop
         id, value = db.get_first_row("SELECT * FROM queues LIMIT 1;")
         db.execute("DELETE FROM queues WHERE id = ?", id)
+        self.count_pop += 1
         value
       end
     

data/lib/sparrow/runner.rb

@@ -16,16 +16,27 @@ module Sparrow
         :host => "0.0.0.0",
         :port => 11212,
         :debug => false,
-        :base_dir => base_dir,
-        :pid_path => pid_path,
-        :log_path => log_path,
-        :type => 'disk'
+        :type => 'disk',
+        :start_time => Time.now
       }
       
+      self.options.merge!({
+        :base_dir => base_dir,
+        :pid_dir => pid_dir,
+        :log_path => log_path
+      })
+      
       parse_options
       
+      FileUtils.mkdir_p(options[:base_dir])
+      FileUtils.mkdir_p(File.dirname(options[:log_path]))
+      
       if options.include?(:kill)
-        kill_pid(options[:kill] || '*')
+        kill_pid(options[:port])
+      end
+      
+      if options.include?(:kill_all)
+        kill_pid('*')
       end
       
       if !options[:daemonize]
@@ -89,18 +100,22 @@ module Sparrow
         
         opts.separator ""; opts.separator "Daemonization:"
         
-        opts.on("-P", "--pid FILE", String, "save PID in FILE when using -d option.", "(default: #{options[:pid_path]})") do |v|
-          options[:pid_path] = File.expand_path(v)
+        opts.on("-P", "--pid FILE", String, "save PID in DIR when using -d option.", "(default: #{options[:pid_dir]})") do |v|
+          options[:pid_dir] = File.expand_path(v)
         end
         
         opts.on("-d", "--daemon", "Daemonize mode") do |v|
           options[:daemonize] = v
         end
 
-        opts.on("-k", "--kill PORT", String, "Kill specified running daemons - leave blank to kill all.") do |v|
+        opts.on("-k", "--kill", "Kill daemons on port #{options[:port]}.") do |v|
           options[:kill] = v
         end
         
+        opts.on("-j", "--kill-all", String, "Kill specified running daemons - leave blank to kill all.") do |v|
+          options[:kill_all] = v
+        end
+        
         opts.separator ""; opts.separator "Logging:"
         
         opts.on("-l", "--log [FILE]", String, "Path to print debugging information.") do |v|
@@ -129,12 +144,12 @@ module Sparrow
     private
     
     def store_pid(pid)
-     FileUtils.mkdir_p(File.dirname(pid_path))
-     File.open(pid_path, 'w'){|f| f.write("#{pid}\n")}
+     FileUtils.mkdir_p(pid_dir)
+     File.open(File.join(pid_dir, "sparrow.#{options[:port]}.pid"), 'w'){|f| f.write("#{pid}\n") }
     end
 
     def kill_pid(k)
-      Dir[options[:pid_path]||File.join(File.dirname(pid_dir), "sparrow.#{k}.pid")].each do |f|
+      Dir[File.join(pid_dir, "sparrow.#{k}.pid")].each do |f|
         begin
         puts f
         pid = IO.read(f).chomp.to_i

data/lib/sparrow/server.rb

@@ -1,3 +1,4 @@
+require 'socket'
 module Sparrow
   module Server
     include Sparrow::Miscel
@@ -32,6 +33,8 @@ module Sparrow
   
     VERSION         = "VERSION"
     
+    STATS           = "STATS"
+    
     SET_REGEX       = /\ASET\s/i
     ADD_REGEX       = /\AADD\s/i
     REPLACE_REGEX   = /\AREPLACE\s/i
@@ -40,6 +43,21 @@ module Sparrow
     QUIT_REGEX      = /\AQUIT/i
     FLUSH_ALL_REGEX = /\AFLUSH_ALL/i
     VERSION_REGEX   = /\AVERSION/i
+    STATS_REGEX     = /\ASTATS/i
+    
+    mattr_accessor :bytes_read
+    mattr_accessor :bytes_written
+    mattr_accessor :connections_made
+    mattr_accessor :connections_lost
+    mattr_accessor :get_count
+    mattr_accessor :set_count
+    
+    self.bytes_read = 0
+    self.bytes_written = 0
+    self.connections_made = 0
+    self.connections_lost = 0
+    self.get_count = 0
+    self.set_count = 0
 
     def post_init
       @current_queue = nil
@@ -47,11 +65,23 @@ module Sparrow
       @expected_bytes = 0
       @current_flag = nil
       @buffer = ''
-      logger.debug "New client"
+      self.connections_made += 1
+      logger.debug "New client [#{client_ip}]"
+    end
+    
+    def unbind
+      self.connections_lost += 1
+      logger.debug "Lost client"
+    end
+    
+    def send_data(data)
+      self.bytes_written += 1
+      super(data)
     end
     
     def receive_data(data)
       logger.debug "Receiving data: #{data}"
+      self.bytes_read += 1
       @buffer << data
       @buffer = process_whole_messages(@buffer)
     end
@@ -89,6 +119,8 @@ module Sparrow
         version_command
       elsif ln =~ FLUSH_ALL_REGEX
         flush_all_command
+      elsif ln =~ STATS_REGEX
+        stats_command
       elsif @expecting_body
         process_body
       else
@@ -102,7 +134,6 @@ module Sparrow
       publish CLIENT_ERROR, e
       publish ERROR
     rescue => e
-      debugger
       logger.error e
       publish SERVER_ERROR, e
     end
@@ -131,6 +162,7 @@ module Sparrow
       @data << @current_flag
       logger.debug "Adding message to queue - #{@current_queue}"
       Sparrow::Queue.add_message(@current_queue, @data)
+      self.set_count += 1
       @expected_bytes = 0
       @current_queue = nil
       @expecting_body = false
@@ -145,8 +177,8 @@ module Sparrow
       raise ClientError if args.empty?
       rsp = []
       args.each do |queue|
+        logger.debug "Getting message from queue - #{queue}"
         begin
-          logger.debug "Getting message from queue - #{queue}"
           msg = Sparrow::Queue.next_message(queue)
           next unless msg
         rescue NoMoreMessages
@@ -156,6 +188,7 @@ module Sparrow
         msg = msg[0..-2]
         rsp << [VALUE, queue, flag, msg.length].join(' ')
         rsp << msg
+        self.get_count += 1
       end
       rsp << EOF
       send_data(rsp.join(CR) + CR)
@@ -172,6 +205,24 @@ module Sparrow
         publish NOT_FOUND
       end
     end
+    
+    def stats_command
+      rsp = []
+      stats_hash = Sparrow::Queue.get_stats(args[1])
+      stats_hash.merge!({
+        :curr_connections   => (self.connections_made - self.connections_lost),
+        :total_connections  => self.connections_made,
+        :bytes_read         => self.bytes_read,
+        :bytes_written      => self.bytes_written,
+        :get_count          => self.get_count,
+        :set_count          => self.set_count
+      })
+      stats_hash.each do |key, value|
+        rsp << [STATS, key, value].join(' ')
+      end
+      rsp << EOF
+      send_data(rsp.join(CR) + CR)
+    end
   
     # FLUSH_ALL
     def flush_all_command
@@ -196,6 +247,10 @@ module Sparrow
     def args
       @split_args ||= @data.split(' ')
     end
+    
+    def client_ip
+      Socket.unpack_sockaddr_in(get_peername)[1]
+    end
   
   end
 end

data/lib/sparrow/utils.rb

@@ -42,6 +42,51 @@ class Class # :nodoc:
   end
 end
 
+class Module # :nodoc:
+  def mattr_reader(*syms)
+    syms.each do |sym|
+      next if sym.is_a?(Hash)
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}
+          @@#{sym} = nil
+        end
+        
+        def self.#{sym}
+          @@#{sym}
+        end
+
+        def #{sym}
+          @@#{sym}
+        end
+      EOS
+    end
+  end
+  
+  def mattr_writer(*syms)
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}
+          @@#{sym} = nil
+        end
+        
+        def self.#{sym}=(obj)
+          @@#{sym} = obj
+        end
+        
+        def #{sym}=(obj)
+          @@#{sym} = obj
+        end
+      EOS
+    end
+  end
+  
+  def mattr_accessor(*syms)
+    mattr_reader(*syms)
+    mattr_writer(*syms)
+  end
+end
+
+
 module Sparrow
   module Miscel
     def options
@@ -60,8 +105,8 @@ module Sparrow
       Sparrow.log_path
     end
   
-    def pid_path
-      Sparrow.log_path
+    def pid_dir
+      Sparrow.pid_dir
     end
     
     def logger

metadata

@@ -3,15 +3,15 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: sparrow
 version: !ruby/object:Gem::Version 
-  version: "0.2"
-date: 2008-01-26 00:00:00 +00:00
+  version: "0.3"
+date: 2008-02-02 00:00:00 +00:00
 summary: Simple file based messagine queue using the memcache protocol
 require_paths: 
 - lib
 email: info@eribium.org
 homepage: http://code.google.com/p/sparrow
 rubyforge_project: Sparrow
-description: The author was too lazy to write a description
+description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached.  # That means you can use Sparrow with any memcached client library (Ruby or otherwise).  #  # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second.  # The load Sparrow can cope with increases exponentially as you add to the cluster.  # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. #  # Sparrow comes with built in support for daemonization and clustering.  # Also included are example libraries and clients. For example: #  # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name']       #=> 1 Pull next msg from queue # m['queue_name']       #=> nil # m.delete('queue_name) # Delete queue #  # # or using the included client: #  # class MyQueue < MQ3::Queue #   def on_message #     logger.info \"Received msg with args: #{args.inspect}\" #   end # end #  # MyQueue.servers = [ #   MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run #  # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't  # be the same order when they're removed. #  # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. #  # Call sparrow with --help for usage options #  # The daemonization won't work on Windows.  #  # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow #  # Sparrow was inspired by Twitter's Starling"
 autorequire: 
 default_executable: 
 bindir: bin
@@ -74,5 +74,5 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.4.0
+        version: 1.5.0
     version: