starling 0.9.3

data/History.txt

@@ -0,0 +1,3 @@
+== 1.0.0 2007-11-02
+
+* Initial release

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Blaine Cook, Twitter, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,29 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/starling
+config/hoe.rb
+config/requirements.rb
+lib/starling.rb
+lib/starling/handler.rb
+lib/starling/persistent_queue.rb
+lib/starling/queue_collection.rb
+lib/starling/runner.rb
+lib/starling/server.rb
+script/destroy
+script/generate
+script/txt2html
+setup.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/website.rake
+test/test_helper.rb
+test/test_persistent_queue.rb
+test/test_starling.rb
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.rhtml

data/README.txt

@@ -0,0 +1,42 @@
+= Name
+
+Starling - a light weight server for reliable distributed message passing.
+
+= Synopsis
+
+  # Start the Starling server as a daemonized process:
+  starling -h 192.168.1.1 -d
+
+  # Put messages onto a queue:
+  require 'memcache'
+  starling = MemCache.new('192.168.1.1:22122')
+  starling.set('my_queue', 12345)
+
+  # Get messages from the queue:
+  require 'memcache'
+  starling = MemCache.new('192.168.1.1:22122')
+  loop { puts starling.get('my_queue') }
+
+  # See the Starling documentation for more information.
+
+= Description
+
+Starling is a powerful but simple messaging server that enables reliable 
+distributed queuing with an absolutely minimal overhead. It speaks the
+MemCache protocol for maximum cross-platform compatibility. Any language
+that speaks MemCache can take advantage of Starling's queue facilities.
+
+= Known Issues
+
+* Starling is "slow" as far as messaging systems are concerned. In practice,
+  it's fast enough. If you'd like to help make it faster please do. Starting
+  points would be to use an event-driven interface, and get rid of threading.
+
+= Authors
+
+Blaine Cook <romeda@gmail.com>
+
+= Copyright
+
+Starling - a light-weight server for reliable distributed message passing.
+Copyright 2007 Blaine Cook <blaine@twitter.com>, Twitter Inc.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/bin/starling

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'starling/runner'
+StarlingServer::Runner.run

data/config/hoe.rb

@@ -0,0 +1,71 @@
+require 'starling/server'
+
+AUTHOR            = 'Blaine Cook'  # can also be an array of Authors
+EMAIL             = "blaine@twitter.com"
+DESCRIPTION       = "Starling is a lightweight, transactional, distributed queue server"
+GEM_NAME          = 'starling' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'starling' # The unix name for your project
+HOMEPATH          = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH     = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file      = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = StarlingServer::VERSION + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'starling documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
+  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,15 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))

data/lib/starling.rb

@@ -0,0 +1,104 @@
+require 'memcache'
+
+class Starling < MemCache
+
+  WAIT_TIME = 0.25
+
+  ##
+  # fetch an item from a queue.
+  
+  def get(*args)
+    loop do
+      response = super(*args)
+      return response unless response.nil?
+      sleep WAIT_TIME
+    end
+  end
+
+  ##
+  # insert +value+ into +queue+. 
+  #
+  # +expiry+ is expressed as a UNIX timestamp
+  # 
+  # If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
+  # will be serialized with YAML, instead.
+  
+  def set(queue, value, expiry = 0, raw = false)
+    retries = 0
+    begin
+      if raw == :yaml
+        value = YAML.dump(value)
+        raw = true
+      end
+
+      super(queue, value, expiry, raw)
+    rescue MemCache::MemCacheError => e
+      retries += 1
+      sleep WAIT_TIME
+      retry unless retries > 3
+      raise e
+    end
+  end
+
+  ##
+  # returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
+  # queue sizes will be returned.
+
+  def sizeof(queue, statistics = nil)
+    statistics ||= stats
+
+    if queue == :all
+      queue_sizes = {}
+      available_queues(statistics).each do |queue|
+        queue_sizes[queue] = sizeof(queue, statistics)
+      end
+      return queue_sizes
+    end
+
+    statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
+  end
+
+  ##
+  # returns a list of available (currently allocated) queues.
+
+  def available_queues(statistics = nil)
+    statistics ||= stats
+
+    statistics.map { |k,v|
+      v.keys
+    }.flatten.uniq.grep(/^queue_(.*)_items/).map { |v|
+      v.gsub(/^queue_/, '').gsub(/_items$/, '')
+    }.reject { |v|
+      v =~ /_total$/ || v =~ /_expired$/
+    }
+  end
+
+  ##
+  # iterator to flush +queue+. Each element will be passed to the provided
+  # +block+
+
+  def flush(queue)
+    sizeof(queue).times do
+      v = get(queue)
+      yield v if block_given?
+    end
+  end
+
+  private
+
+  def get_server_for_key(key)
+    raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
+    raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
+    raise MemCacheError, "No servers available" if @servers.empty?
+
+    bukkits = @buckets.dup
+    bukkits.nitems.times do |try|
+      n = rand(bukkits.nitems)
+      server = bukkits[n]
+      return server if server.alive?
+      bukkits.delete_at(n)
+    end
+
+    raise MemCacheError, "No servers available (all dead)"
+  end
+end

data/lib/starling/handler.rb

@@ -0,0 +1,181 @@
+module StarlingServer
+
+  ##
+  # This is an internal class that's used by Starling::Server to handle the
+  # MemCache protocol and act as an interface between the Server and the
+  # QueueCollection.
+  
+  class Handler
+
+    DATA_PACK_FMT = "Ia*".freeze
+
+    # ERROR responses
+    ERR_UNKNOWN_COMMAND = "CLIENT_ERROR bad command line format\r\n".freeze
+
+    # GET Responses
+    GET_COMMAND = /^get (.{1,250})\r\n$/
+    GET_RESPONSE       = "VALUE %s %s %s\r\n%s\r\nEND\r\n".freeze
+    GET_RESPONSE_EMPTY = "END\r\n".freeze
+
+    # SET Responses
+    SET_COMMAND = /^set (.{1,250}) ([0-9]+) ([0-9]+) ([0-9]+)\r\n$/
+    SET_RESPONSE_SUCCESS  = "STORED\r\n".freeze
+    SET_RESPONSE_FAILURE  = "NOT STORED\r\n".freeze
+    SET_CLIENT_DATA_ERROR = "CLIENT_ERROR bad data chunk\r\nERROR\r\n".freeze
+
+    # STAT Response
+    STATS_COMMAND = /stats\r\n$/
+    STATS_RESPONSE = "STAT pid %d
+STAT uptime %d
+STAT time %d
+STAT version %s
+STAT rusage_user %0.6f
+STAT rusage_system %0.6f
+STAT curr_items %d
+STAT total_items %d
+STAT bytes %d
+STAT curr_connections %d
+STAT total_connections %d
+STAT cmd_get %d
+STAT cmd_set %d
+STAT get_hits %d
+STAT get_misses %d
+STAT bytes_read %d
+STAT bytes_written %d
+STAT limit_maxbytes %d
+%sEND\r\n".freeze
+    QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d
+STAT queue_%s_total_items %d
+STAT queue_%s_logsize %d
+STAT queue_%s_expired_items %d\n".freeze
+
+    ##
+    # Creates a new handler for the MemCache protocol that communicates with a
+    # given client.
+
+    def initialize(client, server, queue_collection)
+      @client = client
+      @server = server
+      @queue_collection = queue_collection
+      @expiry_stats = Hash.new(0)
+    end
+
+    ##
+    # Process incoming commands from the attached client.
+
+    def run
+      while running? do
+        process_command(@client.readline)
+        Thread.current[:last_activity] = Time.now
+      end
+    end
+
+    private
+
+    def running?
+      !Thread.current[:shutdown]
+    end
+
+    def respond(str, *args)
+      response = sprintf(str, *args)
+      @server.stats[:bytes_written] += response.length
+      @client.write response
+    end
+
+    def process_command(command)
+      begin
+        @server.stats[:bytes_read] += command.length
+        case command
+        when SET_COMMAND
+          @server.stats[:set_requests] += 1
+          set($1, $2, $3, $4.to_i)
+        when GET_COMMAND
+          @server.stats[:get_requests] += 1
+          get($1)
+        when STATS_COMMAND
+          stats
+        else
+          logger.warn "Unknown command: #{command[0,4]}.\nFull command was #{command}."
+          respond ERR_UNKNOWN_COMMAND
+        end
+      rescue => e
+        logger.error "Error handling request: #{e}."
+        logger.debug e.backtrace.join("\n")
+        respond GET_RESPONSE_EMPTY
+      end
+    end
+
+    def set(key, flags, expiry, len)
+      data = @client.read(len)
+      data_end = @client.read(2)
+      @server.stats[:bytes_read] += len + 2
+      if data_end == "\r\n" && data.size == len
+        internal_data = [expiry.to_i, data].pack(DATA_PACK_FMT)
+        if @queue_collection.put(key, internal_data)
+          respond SET_RESPONSE_SUCCESS
+        else
+          respond SET_RESPONSE_FAILURE
+        end
+      else
+        respond SET_CLIENT_DATA_ERROR
+      end
+    end
+
+    def get(key)
+      now = Time.now.to_i
+
+      while response = @queue_collection.take(key)
+        expiry, data = response.unpack(DATA_PACK_FMT)
+
+        break if expiry == 0 || expiry >= now
+
+        @expiry_stats[key] += 1
+        expiry, data = nil
+      end
+
+      if data
+        respond GET_RESPONSE, key, 0, data.size, data
+      else
+        respond GET_RESPONSE_EMPTY
+      end
+    end
+
+    def stats
+      respond STATS_RESPONSE, 
+        Process.pid, # pid
+        Time.now - @server.stats(:start_time), # uptime
+        Time.now.to_i, # time
+        StarlingServer::VERSION, # version
+        Process.times.utime, # rusage_user
+        Process.times.stime, # rusage_system
+        @queue_collection.stats(:current_size), # curr_items
+        @queue_collection.stats(:total_items), # total_items
+        @queue_collection.stats(:current_bytes), # bytes
+        @server.stats(:connections), # curr_connections
+        @server.stats(:total_connections), # total_connections
+        @server.stats(:get_requests), # get count
+        @server.stats(:set_requests), # set count
+        @queue_collection.stats(:get_hits),
+        @queue_collection.stats(:get_misses),
+        @server.stats(:bytes_read), # total bytes read
+        @server.stats(:bytes_written), # total bytes written
+        0, # limit_maxbytes
+        queue_stats
+    end
+
+    def queue_stats
+      @queue_collection.queues.inject("") do |m,(k,v)|
+        m + sprintf(QUEUE_STATS_RESPONSE,
+                      k, v.length,
+                      k, v.total_items,
+                      k, v.logsize,
+                      k, @expiry_stats[k])
+      end
+    end
+
+    def logger
+      @server.logger
+    end
+
+  end
+end