timshadel-starling 0.9.8.01.20080924

data/lib/starling.rb

@@ -0,0 +1,131 @@
+require 'memcache'
+
+class Starling < MemCache
+
+  WAIT_TIME = 0.25
+  alias_method :_original_get, :get
+  alias_method :_original_delete, :delete
+
+  ##
+  # fetch an item from a queue.
+
+  def get(*args)
+    loop do
+      response = _original_get(*args)
+      return response unless response.nil?
+      sleep WAIT_TIME
+    end
+  end
+  
+  ##
+  # will return the next item or nil
+
+  def fetch(*args)
+    _original_get(*args)
+  end
+
+  ##
+  # Delete the key (queue) from all Starling servers. This is necessary
+  # because the random way a server is chosen in #get_server_for_key
+  # implies that the queue could easily be spread across the entire
+  # Starling cluster.
+
+  def delete(key, expiry = 0)
+    raise MemCacheError, "Update of readonly cache" if @readonly
+    cache_key = make_cache_key key
+ 
+    @buckets.each do |server|
+      with_socket_management(server) do |socket|
+        socket.write "delete #{cache_key} #{expiry}\r\n"
+        socket.gets
+      end
+    end
+  end
+  
+  ##
+  # insert +value+ into +queue+.
+  #
+  # +expiry+ is expressed as a UNIX timestamp
+  #
+  # If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
+  # will be serialized with YAML, instead.
+
+  def set(queue, value, expiry = 0, raw = false)
+    retries = 0
+    begin
+      if raw == :yaml
+        value = YAML.dump(value)
+        raw = true
+      end
+
+      super(queue, value, expiry, raw)
+    rescue MemCache::MemCacheError => e
+      retries += 1
+      sleep WAIT_TIME
+      retry unless retries > 3
+      raise e
+    end
+  end
+
+  ##
+  # returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
+  # queue sizes will be returned.
+
+  def sizeof(queue, statistics = nil)
+    statistics ||= stats
+
+    if queue == :all
+      queue_sizes = {}
+      available_queues(statistics).each do |queue|
+        queue_sizes[queue] = sizeof(queue, statistics)
+      end
+      return queue_sizes
+    end
+
+    statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
+  end
+
+  ##
+  # returns a list of available (currently allocated) queues.
+
+  def available_queues(statistics = nil)
+    statistics ||= stats
+
+    statistics.map { |k,v|
+      v.keys
+    }.flatten.uniq.grep(/^queue_(.*)_items/).map { |v|
+      v.gsub(/^queue_/, '').gsub(/_items$/, '')
+    }.reject { |v|
+      v =~ /_total$/ || v =~ /_expired$/
+    }
+  end
+
+  ##
+  # iterator to flush +queue+. Each element will be passed to the provided
+  # +block+
+
+  def flush(queue)
+    sizeof(queue).times do
+      v = get(queue)
+      yield v if block_given?
+    end
+  end
+
+  private
+
+  def get_server_for_key(key)
+    raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
+    raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
+    raise MemCacheError, "No servers available" if @servers.empty?
+
+    bukkits = @buckets.dup
+    bukkits.nitems.times do |try|
+      n = rand(bukkits.nitems)
+      server = bukkits[n]
+      return server if server.alive?
+      bukkits.delete_at(n)
+    end
+
+    raise MemCacheError, "No servers available (all dead)"
+  end
+end

data/spec/starling_server_spec.rb

@@ -0,0 +1,216 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'fileutils'
+require 'memcache'
+require 'digest/md5'
+require 'starling'
+
+require 'starling/server'
+
+class StarlingServer::PersistentQueue
+  remove_const :SOFT_LOG_MAX_SIZE
+  SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB
+end
+
+def safely_fork(&block)
+  # anti-race juice:
+  blocking = true
+  Signal.trap("USR1") { blocking = false }
+  
+  pid = Process.fork(&block)
+  
+  while blocking
+    sleep 0.1
+  end
+
+  pid
+end
+
+describe "StarlingServer" do
+  before do
+    @tmp_path = File.join(File.dirname(__FILE__), "tmp")
+    
+    begin
+      Dir::mkdir(@tmp_path)
+    rescue Errno::EEXIST
+    end
+
+    @server_pid = safely_fork do
+      server = StarlingServer::Base.new(:host => '127.0.0.1',
+                                        :port => 22133,
+                                        :path => @tmp_path,
+                                        :logger => Logger.new(STDERR),
+                                        :log_level => Logger::FATAL)
+      Signal.trap("INT") {
+        server.stop
+        exit
+      }
+
+      Process.kill("USR1", Process.ppid)
+      server.run
+    end
+
+    @client = MemCache.new('127.0.0.1:22133')
+
+  end
+  
+  it "should test if temp_path exists and is writeable" do
+    File.exist?(@tmp_path).should be_true
+    File.directory?(@tmp_path).should be_true
+    File.writable?(@tmp_path).should be_true
+  end
+
+  it "should set and get" do
+    v = rand((2**32)-1)
+    @client.get('test_set_and_get_one_entry').should be_nil
+    @client.set('test_set_and_get_one_entry', v)
+    @client.get('test_set_and_get_one_entry').should eql(v)
+  end
+  
+  it "should respond to delete" do
+    @client.delete("my_queue").should eql("END\r\n")   
+    starling_client = Starling.new('127.0.0.1:22133')
+    starling_client.set('my_queue', 50)
+    starling_client.available_queues.size.should eql(1)
+    starling_client.delete("my_queue")
+    starling_client.available_queues.size.should eql(0)
+  end
+  
+  it "should expire entries" do
+    v = rand((2**32)-1)
+    @client.get('test_set_with_expiry').should be_nil
+    now = Time.now.to_i
+    @client.set('test_set_with_expiry', v + 2, now)
+    @client.set('test_set_with_expiry', v)
+    sleep(1.0)
+    @client.get('test_set_with_expiry').should eql(v)
+  end
+
+  it "should have age stat" do
+    now = Time.now.to_i
+    @client.set('test_age', 'nibbler')
+    sleep(1.0)
+    @client.get('test_age').should eql('nibbler')
+
+    stats = @client.stats['127.0.0.1:22133']
+    stats.has_key?('queue_test_age_age').should be_true
+    (stats['queue_test_age_age'] >= 1000).should be_true
+  end
+  
+  it "should rotate log" do
+    log_rotation_path = File.join(@tmp_path, 'test_log_rotation')
+
+    Dir.glob("#{log_rotation_path}*").each do |file|
+      File.unlink(file) rescue nil
+    end
+    @client.get('test_log_rotation').should be_nil
+
+    v = 'x' * 8192
+
+    @client.set('test_log_rotation', v)
+    File.size(log_rotation_path).should eql(8207)
+    @client.get('test_log_rotation')
+
+    @client.get('test_log_rotation').should be_nil
+
+    @client.set('test_log_rotation', v)
+    @client.get('test_log_rotation').should eql(v)
+
+    File.size(log_rotation_path).should eql(1)
+    # rotated log should be erased after a successful roll.
+    Dir.glob("#{log_rotation_path}*").size.should eql(1)
+  end
+
+  it "should output statistics per server" do
+    stats = @client.stats
+    assert_kind_of Hash, stats
+    assert stats.has_key?('127.0.0.1:22133')
+    
+    server_stats = stats['127.0.0.1:22133']
+    
+    basic_stats = %w( bytes pid time limit_maxbytes cmd_get version
+                      bytes_written cmd_set get_misses total_connections
+                      curr_connections curr_items uptime get_hits total_items
+                      rusage_system rusage_user bytes_read )
+
+    basic_stats.each do |stat|
+      server_stats.has_key?(stat).should be_true
+    end
+  end
+
+  it "should return valid response with unkown command" do
+    response = @client.add('blah', 1)
+    response.should eql("CLIENT_ERROR bad command line format\r\n")
+  end
+
+  it "should disconnect and reconnect again" do
+    v = rand(2**32-1)
+    @client.set('test_that_disconnecting_and_reconnecting_works', v)
+    @client.reset
+    @client.get('test_that_disconnecting_and_reconnecting_works').should eql(v)
+  end
+  
+  it "should use epoll on linux" do
+    # this may take a few seconds.
+    # the point is to make sure that we're using epoll on Linux, so we can
+    # handle more than 1024 connections.
+    
+    unless IO::popen("uname").read.chomp == "Linux"
+      pending "skipping epoll test: not on Linux"
+    end
+
+    fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i
+    unless fd_limit > 1024
+      pending "skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024"
+    end
+    
+    v = rand(2**32 - 1)
+    @client.set('test_epoll', v)
+
+    # we can't open 1024 connections to memcache from within this process,
+    # because we will hit ruby's 1024 fd limit ourselves!
+    pid1 = safely_fork do
+      unused_sockets = []
+      600.times do
+        unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+      end
+      Process.kill("USR1", Process.ppid)
+      sleep 90
+    end
+    pid2 = safely_fork do
+      unused_sockets = []
+      600.times do
+        unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+      end
+      Process.kill("USR1", Process.ppid)
+      sleep 90
+    end
+
+    begin
+      client = MemCache.new('127.0.0.1:22133')
+      client.get('test_epoll').should eql(v)
+    ensure
+      Process.kill("TERM", pid1)
+      Process.kill("TERM", pid2)
+    end
+  end
+  
+  it "should raise error if queue collection is an invalid path" do
+    invalid_path = nil
+    while invalid_path.nil? || File.exist?(invalid_path)
+      invalid_path = File.join('/', Digest::MD5.hexdigest(rand(2**32-1).to_s)[0,8])
+    end
+
+    lambda {
+      StarlingServer::QueueCollection.new(invalid_path)
+    }.should raise_error(StarlingServer::InaccessibleQueuePath)
+  end
+  
+  after do
+    Process.kill("INT", @server_pid)
+    Process.wait(@server_pid)
+    @client.reset
+    FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification 
+name: timshadel-starling
+version: !ruby/object:Gem::Version 
+  version: 0.9.8.01.20080924
+platform: ruby
+authors: 
+- Blaine Cook
+- Chris Wanstrath
+- Britt Selvitelle
+- Glenn Rempe
+- Abdul-Rahman Advany
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-08-13 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: memcache-client
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.0
+    version: 
+description: Starling is a lightweight, transactional, distributed queue server
+email: 
+- blaine@twitter.com
+- chris@ozmm.org
+- abdulrahman@advany.com
+executables: 
+- starling
+- starling_top
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- CHANGELOG
+- LICENSE
+files: 
+- README.rdoc
+- LICENSE
+- CHANGELOG
+- Rakefile
+- lib/starling/handler.rb
+- lib/starling/persistent_queue.rb
+- lib/starling/queue_collection.rb
+- lib/starling/server_runner.rb
+- lib/starling/server.rb
+- lib/starling.rb
+- etc/starling.redhat
+- etc/starling.ubuntu
+- etc/sample-config.yml
+has_rdoc: true
+homepage: http://github.com/starling/starling/
+post_install_message: 
+rdoc_options: 
+- --quiet
+- --title
+- starling documentation
+- --opname
+- index.html
+- --line-numbers
+- --main
+- README.rdoc
+- --inline-source
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Starling is a lightweight, transactional, distributed queue server
+test_files: 
+- spec/starling_server_spec.rb