starling 0.9.3 → 0.9.8

data/spec/starling_server_spec.rb

@@ -0,0 +1,205 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'fileutils'
+require 'memcache'
+require 'digest/md5'
+
+require 'starling/server'
+
+class StarlingServer::PersistentQueue
+  remove_const :SOFT_LOG_MAX_SIZE
+  SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB
+end
+
+def safely_fork(&block)
+  # anti-race juice:
+  blocking = true
+  Signal.trap("USR1") { blocking = false }
+  
+  pid = Process.fork(&block)
+  
+  while blocking
+    sleep 0.1
+  end
+
+  pid
+end
+
+describe "StarlingServer" do
+  before do
+    @tmp_path = File.join(File.dirname(__FILE__), "tmp")
+    
+    begin
+      Dir::mkdir(@tmp_path)
+    rescue Errno::EEXIST
+    end
+
+    @server_pid = safely_fork do
+      server = StarlingServer::Base.new(:host => '127.0.0.1',
+                                        :port => 22133,
+                                        :path => @tmp_path,
+                                        :logger => Logger.new(STDERR),
+                                        :log_level => Logger::FATAL)
+      Signal.trap("INT") {
+        server.stop
+        exit
+      }
+
+      Process.kill("USR1", Process.ppid)
+      server.run
+    end
+
+    @client = MemCache.new('127.0.0.1:22133')
+  end
+  
+  it "should test if temp_path exists and is writeable" do
+    File.exist?(@tmp_path).should be_true
+    File.directory?(@tmp_path).should be_true
+    File.writable?(@tmp_path).should be_true
+  end
+
+  it "should set and get" do
+    v = rand((2**32)-1)
+    @client.get('test_set_and_get_one_entry').should be_nil
+    @client.set('test_set_and_get_one_entry', v)
+    @client.get('test_set_and_get_one_entry').should eql(v)
+  end
+  
+  it "should expire entries" do
+    v = rand((2**32)-1)
+    @client.get('test_set_with_expiry').should be_nil
+    now = Time.now.to_i
+    @client.set('test_set_with_expiry', v + 2, now)
+    @client.set('test_set_with_expiry', v)
+    sleep(1.0)
+    @client.get('test_set_with_expiry').should eql(v)
+  end
+
+  it "should have age stat" do
+    now = Time.now.to_i
+    @client.set('test_age', 'nibbler')
+    sleep(1.0)
+    @client.get('test_age').should eql('nibbler')
+
+    stats = @client.stats['127.0.0.1:22133']
+    stats.has_key?('queue_test_age_age').should be_true
+    (stats['queue_test_age_age'] >= 1000).should be_true
+  end
+  
+  it "should rotate log" do
+    log_rotation_path = File.join(@tmp_path, 'test_log_rotation')
+
+    Dir.glob("#{log_rotation_path}*").each do |file|
+      File.unlink(file) rescue nil
+    end
+    @client.get('test_log_rotation').should be_nil
+
+    v = 'x' * 8192
+
+    @client.set('test_log_rotation', v)
+    File.size(log_rotation_path).should eql(8207)
+    @client.get('test_log_rotation')
+
+    @client.get('test_log_rotation').should be_nil
+
+    @client.set('test_log_rotation', v)
+    @client.get('test_log_rotation').should eql(v)
+
+    File.size(log_rotation_path).should eql(1)
+    # rotated log should be erased after a successful roll.
+    Dir.glob("#{log_rotation_path}*").size.should eql(1)
+  end
+
+  it "should output statistics per server" do
+    stats = @client.stats
+    assert_kind_of Hash, stats
+    assert stats.has_key?('127.0.0.1:22133')
+    
+    server_stats = stats['127.0.0.1:22133']
+    
+    basic_stats = %w( bytes pid time limit_maxbytes cmd_get version
+                      bytes_written cmd_set get_misses total_connections
+                      curr_connections curr_items uptime get_hits total_items
+                      rusage_system rusage_user bytes_read )
+
+    basic_stats.each do |stat|
+      server_stats.has_key?(stat).should be_true
+    end
+  end
+
+  it "should return valid response with unkown command" do
+    response = @client.add('blah', 1)
+    response.should eql("CLIENT_ERROR bad command line format\r\n")
+  end
+
+  it "should disconnect and reconnect again" do
+    v = rand(2**32-1)
+    @client.set('test_that_disconnecting_and_reconnecting_works', v)
+    @client.reset
+    @client.get('test_that_disconnecting_and_reconnecting_works').should eql(v)
+  end
+  
+  it "should use epoll on linux" do
+    # this may take a few seconds.
+    # the point is to make sure that we're using epoll on Linux, so we can
+    # handle more than 1024 connections.
+    
+    unless IO::popen("uname").read.chomp == "Linux"
+      pending "skipping epoll test: not on Linux"
+    end
+
+    fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i
+    unless fd_limit > 1024
+      pending "skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024"
+    end
+    
+    v = rand(2**32 - 1)
+    @client.set('test_epoll', v)
+
+    # we can't open 1024 connections to memcache from within this process,
+    # because we will hit ruby's 1024 fd limit ourselves!
+    pid1 = safely_fork do
+      unused_sockets = []
+      600.times do
+        unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+      end
+      Process.kill("USR1", Process.ppid)
+      sleep 90
+    end
+    pid2 = safely_fork do
+      unused_sockets = []
+      600.times do
+        unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+      end
+      Process.kill("USR1", Process.ppid)
+      sleep 90
+    end
+
+    begin
+      client = MemCache.new('127.0.0.1:22133')
+      client.get('test_epoll').should eql(v)
+    ensure
+      Process.kill("TERM", pid1)
+      Process.kill("TERM", pid2)
+    end
+  end
+  
+  it "should raise error if queue collection is an invalid path" do
+    invalid_path = nil
+    while invalid_path.nil? || File.exist?(invalid_path)
+      invalid_path = File.join('/', Digest::MD5.hexdigest(rand(2**32-1).to_s)[0,8])
+    end
+
+    lambda {
+      StarlingServer::QueueCollection.new(invalid_path)
+    }.should raise_error(StarlingServer::InaccessibleQueuePath)
+  end
+  
+  after do
+    Process.kill("INT", @server_pid)
+    Process.wait(@server_pid)
+    @client.reset
+    FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
+  end
+end

metadata

@@ -1,66 +1,92 @@
 --- !ruby/object:Gem::Specification 
 name: starling
 version: !ruby/object:Gem::Version 
-  version: 0.9.3
-platform: ""
+  version: 0.9.8
+platform: ruby
 authors: 
 - Blaine Cook
+- Chris Wanstrath
+- Britt Selvitelle
+- Glenn Rempe
+- Abdul-Rahman Advany
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2008-01-10 00:00:00 -08:00
+date: 2008-08-26 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: memcache-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: SyslogLogger
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.0
+    version: 
 description: Starling is a lightweight, transactional, distributed queue server
-email: blaine@twitter.com
+email: 
+- blaine@twitter.com
+- chris@ozmm.org
+- abdulrahman@advany.com
 executables: 
 - starling
+- starling_top
 extensions: []
 
 extra_rdoc_files: 
-- History.txt
-- License.txt
-- Manifest.txt
-- README.txt
-- website/index.txt
+- README.rdoc
+- CHANGELOG
+- LICENSE
 files: 
-- History.txt
-- License.txt
-- Manifest.txt
-- README.txt
+- README.rdoc
+- LICENSE
+- CHANGELOG
 - Rakefile
-- bin/starling
-- config/hoe.rb
-- config/requirements.rb
-- lib/starling.rb
 - lib/starling/handler.rb
 - lib/starling/persistent_queue.rb
 - lib/starling/queue_collection.rb
-- lib/starling/runner.rb
+- lib/starling/server_runner.rb
 - lib/starling/server.rb
-- script/destroy
-- script/generate
-- script/txt2html
-- setup.rb
-- tasks/deployment.rake
-- tasks/environment.rake
-- tasks/website.rake
-- test/test_helper.rb
-- test/test_persistent_queue.rb
-- test/test_starling.rb
-- website/index.html
-- website/index.txt
-- website/javascripts/rounded_corners_lite.inc.js
-- website/stylesheets/screen.css
-- website/template.rhtml
+- lib/starling.rb
+- etc/starling.redhat
+- etc/starling.ubuntu
+- etc/sample-config.yml
 has_rdoc: true
-homepage: http://starling.rubyforge.org
+homepage: http://github.com/starling/starling/
 post_install_message: 
 rdoc_options: 
+- --quiet
+- --title
+- starling documentation
+- --opname
+- index.html
+- --line-numbers
 - --main
-- README.txt
+- README.rdoc
+- --inline-source
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -77,12 +103,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: starling
-rubygems_version: 0.9.5
+rubyforge_project: 
+rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: Starling is a lightweight, transactional, distributed queue server
 test_files: 
-- test/test_helper.rb
-- test/test_persistent_queue.rb
-- test/test_starling.rb
+- spec/starling_server_spec.rb

data/History.txt

@@ -1,3 +0,0 @@
-== 1.0.0 2007-11-02
-
-* Initial release

data/Manifest.txt

@@ -1,29 +0,0 @@
-History.txt
-License.txt
-Manifest.txt
-README.txt
-Rakefile
-bin/starling
-config/hoe.rb
-config/requirements.rb
-lib/starling.rb
-lib/starling/handler.rb
-lib/starling/persistent_queue.rb
-lib/starling/queue_collection.rb
-lib/starling/runner.rb
-lib/starling/server.rb
-script/destroy
-script/generate
-script/txt2html
-setup.rb
-tasks/deployment.rake
-tasks/environment.rake
-tasks/website.rake
-test/test_helper.rb
-test/test_persistent_queue.rb
-test/test_starling.rb
-website/index.html
-website/index.txt
-website/javascripts/rounded_corners_lite.inc.js
-website/stylesheets/screen.css
-website/template.rhtml

data/README.txt

@@ -1,42 +0,0 @@
-= Name
-
-Starling - a light weight server for reliable distributed message passing.
-
-= Synopsis
-
-  # Start the Starling server as a daemonized process:
-  starling -h 192.168.1.1 -d
-
-  # Put messages onto a queue:
-  require 'memcache'
-  starling = MemCache.new('192.168.1.1:22122')
-  starling.set('my_queue', 12345)
-
-  # Get messages from the queue:
-  require 'memcache'
-  starling = MemCache.new('192.168.1.1:22122')
-  loop { puts starling.get('my_queue') }
-
-  # See the Starling documentation for more information.
-
-= Description
-
-Starling is a powerful but simple messaging server that enables reliable 
-distributed queuing with an absolutely minimal overhead. It speaks the
-MemCache protocol for maximum cross-platform compatibility. Any language
-that speaks MemCache can take advantage of Starling's queue facilities.
-
-= Known Issues
-
-* Starling is "slow" as far as messaging systems are concerned. In practice,
-  it's fast enough. If you'd like to help make it faster please do. Starting
-  points would be to use an event-driven interface, and get rid of threading.
-
-= Authors
-
-Blaine Cook <romeda@gmail.com>
-
-= Copyright
-
-Starling - a light-weight server for reliable distributed message passing.
-Copyright 2007 Blaine Cook <blaine@twitter.com>, Twitter Inc.

data/config/hoe.rb

@@ -1,71 +0,0 @@
-require 'starling/server'
-
-AUTHOR            = 'Blaine Cook'  # can also be an array of Authors
-EMAIL             = "blaine@twitter.com"
-DESCRIPTION       = "Starling is a lightweight, transactional, distributed queue server"
-GEM_NAME          = 'starling' # what ppl will type to install your gem
-RUBYFORGE_PROJECT = 'starling' # The unix name for your project
-HOMEPATH          = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
-DOWNLOAD_PATH     = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
-
-@config_file      = "~/.rubyforge/user-config.yml"
-@config = nil
-RUBYFORGE_USERNAME = "unknown"
-def rubyforge_username
-  unless @config
-    begin
-      @config = YAML.load(File.read(File.expand_path(@config_file)))
-    rescue
-      puts <<-EOS
-ERROR: No rubyforge config file found: #{@config_file}
-Run 'rubyforge setup' to prepare your env for access to Rubyforge
- - See http://newgem.rubyforge.org/rubyforge.html for more details
-      EOS
-      exit
-    end
-  end
-  RUBYFORGE_USERNAME.replace @config["username"]
-end
-
-
-REV = nil 
-# UNCOMMENT IF REQUIRED: 
-# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
-VERS = StarlingServer::VERSION + (REV ? ".#{REV}" : "")
-RDOC_OPTS = ['--quiet', '--title', 'starling documentation',
-    "--opname", "index.html",
-    "--line-numbers", 
-    "--main", "README",
-    "--inline-source"]
-
-class Hoe
-  def extra_deps 
-    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
-    @extra_deps
-  end 
-end
-
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-hoe = Hoe.new(GEM_NAME, VERS) do |p|
-  p.author = AUTHOR 
-  p.description = DESCRIPTION
-  p.email = EMAIL
-  p.summary = DESCRIPTION
-  p.url = HOMEPATH
-  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
-  p.test_globs = ["test/**/test_*.rb"]
-  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
-  
-  # == Optional
-  p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
-  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
-  
-  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
-  
-end
-
-CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
-PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
-hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
-hoe.rsync_args = '-av --delete --ignore-errors'