fiveruns-starling 0.9.7.5

data/lib/starling/thread_pool.rb

@@ -0,0 +1,62 @@
+# http://en.wikipedia.org/wiki/Thread_pool_pattern
+class ThreadPool
+
+  # 1. Initialize with arbitrary number of threads 
+  def initialize( threads_number = 1, &exception_handler )
+    @work_queue = SizedQueue.new(threads_number)
+    @worker_threads = ThreadGroup.new
+
+    # Create all threads in advance: that's the thread pool after all
+    threads_number.times do
+
+      worker_thread = Thread.new do
+        # Body of the Worker Thread
+        begin
+          work_to_do = @work_queue.pop() # Blocking
+          break if work_to_do == :END_OF_WORK
+          begin
+            work_to_do.block.call( *work_to_do.args )
+          rescue Exception => e # Since they are swallowed by default
+            if !exception_handler.nil?
+              yield e
+            else
+              # Standard exception handling
+              dump = "Worker Thread thrown an exception:\n"
+              dump += "#{e.message}\n(#{e.class.name})\n"
+              dump += e.backtrace().join( "\n" ) + "\n"
+              print dump
+            end
+          end
+        end until false
+        # Worker thread ends up gracefully <img src='http://ruby-rails.pl/wp-includes/images/smilies/icon_smile.gif' alt=':)' class='wp-smiley' /> 
+      end
+
+      @worker_threads.add worker_thread 
+    end
+  end
+  
+  # 2. Add job to the queue, example:
+  def add_work( *args, &block )
+    @work_queue.push( Runnable.new( args, &block ) )
+  end
+
+  # 3. Allow working threads to end up after finishing all queued work
+  def no_more_work
+    threads_n = @worker_threads.list.length
+    threads_n.times { @work_queue.push :END_OF_WORK }
+  end
+
+  # 4. Wait for all work to finish
+  def join
+    @worker_threads.list.each { |t| t.join }
+  end
+  
+end
+
+class Runnable
+  attr_accessor :args, :block
+  def initialize( args, &block )
+    @args = args
+    @block = block
+  end
+end

data/lib/starling/worker.rb

@@ -0,0 +1,239 @@
+require 'rubygems'
+require 'logger'
+require 'thread'
+require 'timeout'
+require 'starling/thread_pool'
+
+module StarlingWorker
+  attr_reader :logger
+  
+  VERSION = "0.9.7.5"
+  
+  class Base    
+    DEFAULT_TIMEOUT     = 10
+    DEFAULT_CONTINUES_PROCESSING = true
+
+    # you just need to reimplement this
+    def process(message=nil, &block)
+      if block
+        @block = block
+      else
+        raise "you need to implement process"
+      end
+    end
+    
+    def run
+      process_worker(&@block)
+    end
+    
+    def initialize(opts = {})
+      @opts = {
+        :timeout => DEFAULT_TIMEOUT,
+        :incoming_remote_queue_name => nil,
+        :outgoing_remote_queue_name => nil,
+        :continues_processing => DEFAULT_CONTINUES_PROCESSING,
+        :threads => 10,
+        :log_level => Logger::INFO
+      }.merge(opts)
+      
+      @@logger = case @opts[:logger]
+                 when IO, String; Logger.new(@opts[:logger])
+                 when Logger; @opts[:logger]
+                 else; Logger.new(STDERR)
+                 end
+      
+      @@logger = SyslogLogger.new(@opts[:syslog_channel]) if @opts[:syslog_channel]
+
+      @@logger.level = @opts[:log_level] || Logger::ERROR
+
+      @@logger.info "StarlingWorker#{self.class.to_s} STARTUP"
+      
+      unless @opts[:host] && @opts[:port]
+        raise "you need to pass starling host en port"
+      else
+        @@logger.info "StarlingWorker#{self.class.to_s} connecting to starling"
+        @starling = Starling.new("#{@opts[:host]}:#{@opts[:port]}", :multithread => true)
+      end
+      
+      @@logger.info "StarlingWorker#{self.class.to_s} ThreadPool #{@opts[:threads]}"
+      @threadpool = ThreadPool.new(@opts[:threads])
+      
+      @@logger.info "StarlingWorker#{self.class.to_s} Created local queue"
+      @messages = Queue.new #local queue
+    end
+    
+    def process_worker(&block)
+      enq_thread = Thread.new do
+        @@logger.info "StarlingWorker#{self.class.to_s} starting process_message_from_remote_queue"
+        if @opts[:continues_processing]
+          while @opts[:continues_processing]
+            process_message_from_incoming_remote_queue(&block)
+          end
+        else
+          process_message_from_incoming_remote_queue(&block)
+        end
+      end
+      
+      enq_thread.join unless @opts[:continues_processing]
+      
+      deq_thread = Thread.new do
+        @@logger.info "StarlingWorker#{self.class.to_s} starting process_message_from_local_queue_to_outgoing_remote_queue"
+        if @opts[:continues_processing]
+          while @opts[:continues_processing]
+            process_message_from_local_queue_to_outgoing_remote_queue
+          end
+        else
+          process_message_from_local_queue_to_outgoing_remote_queue
+        end
+      end if @opts[:outgoing_remote_queue_name]
+      
+      deq_thread.join unless @opts[:continues_processing] || @opts[:outgoing_remote_queue_name] == nil
+      
+      if @opts[:continues_processing]
+        enq_thread.join
+        deq_thread.join unless @opts[:outgoing_remote_queue_name] == nil
+      end
+    end
+    
+    def process_message_from_incoming_remote_queue(&block)
+      begin
+        if @opts[:incoming_remote_queue_name]
+          starling = Starling.new("#{@opts[:host]}:#{@opts[:port]}", :multithread => true) unless starling
+          message = from_remote_queue_to_process(starling)
+          while message == nil
+            sleep 0.25
+            message = from_remote_queue_to_process(starling)
+          end
+          @@logger.info "StarlingWorker#{self.class.to_s} processing message (#{message})"
+          process_as_thread(message, &block)
+        else
+          @@logger.info "StarlingWorker#{self.class.to_s} processing without message (#{message})"
+          process_as_thread(&block) # just process without message
+        end
+      rescue Exception => e
+        puts "--- process_worker ---"
+        puts e
+        puts e.backtrace.join("\n")
+        sleep 1
+      end
+    end
+    
+    def process_message_from_local_queue_to_outgoing_remote_queue
+      begin
+        starling = Starling.new("#{@opts[:host]}:#{@opts[:port]}", :multithread => true) unless starling
+
+        from_local_queue_to_remote_queue(starling)
+      rescue Exception => e
+        puts "--- process_worker ---"
+        puts e
+        puts e.backtrace.join("\n")
+        sleep 1
+      end
+    end
+    
+    def process_as_thread(message=nil, &block)
+      @threadpool.add_work do
+        timeout(@opts[:timeout]) do
+          if block
+            if message
+              @@logger.info "StarlingWorker#{self.class.to_s} processing block message (#{message})"
+              from_process_to_local_queue block.call(message)
+            else
+              @@logger.info "StarlingWorker#{self.class.to_s} processing block without message (#{message})"
+              from_process_to_local_queue block.call
+            end
+          else
+            if message
+              @@logger.info "StarlingWorker#{self.class.to_s} processing method message (#{message})"
+              from_process_to_local_queue process(message)
+            else
+              @@logger.info "StarlingWorker#{self.class.to_s} processing method without message (#{message})"
+              from_process_to_local_queue process
+            end
+          end
+        end
+      end
+    end
+    
+    def from_remote_queue_to_process(starling=@starling)
+      if @opts[:incoming_remote_queue_name]
+        message = get_message_from_incoming_remote_queue(starling)
+        
+        while message == nil
+          sleep 0.25
+          message = get_message_from_incoming_remote_queue(starling)
+        end
+        
+        @@logger.info "StarlingWorker#{self.class.to_s} return remote message to process (#{message})"
+      
+        message
+      end
+    end
+    
+    def from_process_to_local_queue(message)
+      @@logger.info "StarlingWorker#{self.class.to_s} add message to local queue (#{message})" if @opts[:outgoing_remote_queue_name]
+      add_message_to_local_queue(message) if @opts[:outgoing_remote_queue_name]
+    end
+    
+    def from_local_queue_to_remote_queue(starling=@starling)
+      if @opts[:outgoing_remote_queue_name]
+        message = get_message_from_local_queue
+        
+        @@logger.info "StarlingWorker::#{self.class.to_s} set message on remote queue (#{message})"
+        
+        starling.set(@opts[:outgoing_remote_queue_name], message)
+      end
+    end
+    
+    def threadpool
+      @threadpool
+    end
+    
+    def incoming_remote_queue_name
+      @opts[:incoming_remote_queue_name]
+    end
+    
+    def outgoing_remote_queue_name
+      @opts[:outgoing_remote_queue_name]
+    end
+    
+    def add_message_to_local_queue(message)
+      @messages.enq message
+      true
+    end
+    
+    def get_message_from_local_queue
+      @messages.deq
+    end
+    
+    def local_queue
+      @messages
+    end
+    
+    def add_message_to_incoming_remote_queue(message, starling=@starling)
+      starling.set(incoming_remote_queue_name, message)
+      true
+    end
+    
+    def get_message_from_incoming_remote_queue(starling=@starling)
+      starling.get(incoming_remote_queue_name)
+    end
+    
+    def add_message_to_outgoing_remote_queue(message, starling=@starling)
+      starling.set(outgoing_remote_queue_name, message)
+      true
+    end
+    
+    def get_message_from_outgoing_remote_queue(starling=@starling)
+      starling.get(outgoing_remote_queue_name)
+    end
+    
+    def flush_remote_queue
+      @starling.flush
+    end
+    
+    def self.logger
+      @@logger
+    end
+  end
+end

data/test/test_starling_client.rb

@@ -0,0 +1,100 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'spec'
+require 'fileutils'
+require 'memcache'
+require 'digest/md5'
+
+require 'starling/server'
+require 'starling/client'
+
+class StarlingServer::PersistentQueue
+  remove_const :SOFT_LOG_MAX_SIZE
+  SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB
+end
+
+def safely_fork(&block)
+  # anti-race juice:
+  blocking = true
+  Signal.trap("USR1") { blocking = false }
+  
+  pid = Process.fork(&block)
+  
+  while blocking
+    sleep 0.1
+  end
+  
+  pid
+end
+
+describe "StarlingServer" do
+  before do
+    @tmp_path = File.join(File.dirname(__FILE__), "tmp")
+    @templates_path = File.join(File.dirname(__FILE__), "templates")
+    @workers_path = File.join(File.dirname(__FILE__), "workers")
+    
+    begin
+      Dir::mkdir(@tmp_path)
+    rescue Errno::EEXIST
+    end
+
+    @server_pid = safely_fork do
+      server = StarlingServer::Base.new(:host => '127.0.0.1',
+                                        :port => 22133,
+                                        :path => @tmp_path,
+                                        :logger => Logger.new(STDERR),
+                                        :log_level => Logger::FATAL)
+      Signal.trap("INT") { server.stop }
+      Process.kill("USR1", Process.ppid)
+      server.run
+    end
+    
+    @client = StarlingClient::Base.new(:host => '127.0.0.1',
+                                       :port => 22133,
+                                       :templates_path => @templates_path,
+                                       :workers_path => @workers_path)
+  end
+  
+  it "should test if tmp_path exists and is writeable" do
+    File.exist?(@tmp_path).should be_true
+    File.directory?(@tmp_path).should be_true
+    File.writable?(@tmp_path).should be_true
+  end
+  
+  it "should test if templates_path exists and is writeable" do
+    File.exist?(@templates_path).should be_true
+    File.directory?(@templates_path).should be_true
+    File.writable?(@templates_path).should be_true
+  end
+  
+  it "should test if workers_path exists and is writeable" do
+    File.exist?(@workers_path).should be_true
+    File.directory?(@workers_path).should be_true
+    File.writable?(@workers_path).should be_true
+  end
+  
+  it "should load templates" do
+    @client.load_templates.should eql(["ActiveRecord", "Basic"])
+  end
+  
+  it "should load workers" do
+    @client.load_workers.should eql(["GetDataFromSomeApi", "PushDataToSomeApi"])
+  end
+  
+  it "should return Starling" do
+    @client.starling.should_not == nil
+  end
+  
+  it "should run" do
+    @client.starling.set("get_data_from_some_api_out", "test")
+    @client.starling.set("gettingdata_out", "test")
+    @client.run
+  end
+  
+  after do
+    Process.kill("INT", @server_pid)
+    Process.wait(@server_pid)
+    FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
+  end
+end

data/test/test_starling_server.rb

@@ -0,0 +1,205 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'fileutils'
+require 'memcache'
+require 'digest/md5'
+
+require 'starling/server'
+
+class StarlingServer::PersistentQueue
+  remove_const :SOFT_LOG_MAX_SIZE
+  SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB
+end
+
+def safely_fork(&block)
+  # anti-race juice:
+  blocking = true
+  Signal.trap("USR1") { blocking = false }
+  
+  pid = Process.fork(&block)
+  
+  while blocking
+    sleep 0.1
+  end
+  
+  pid
+end
+
+describe "StarlingServer" do
+  before do
+    @tmp_path = File.join(File.dirname(__FILE__), "tmp")
+    
+    begin
+      Dir::mkdir(@tmp_path)
+    rescue Errno::EEXIST
+    end
+
+    @server_pid = safely_fork do
+      server = StarlingServer::Base.new(:host => '127.0.0.1',
+                                        :port => 22133,
+                                        :path => @tmp_path,
+                                        :logger => Logger.new(STDERR),
+                                        :log_level => Logger::FATAL)
+      Signal.trap("INT") { server.stop }
+      Process.kill("USR1", Process.ppid)
+      server.run
+    end
+
+    @client = MemCache.new('127.0.0.1:22133')
+  end
+  
+  it "should test if temp_path exists and is writeable" do
+    File.exist?(@tmp_path).should be_true
+    File.directory?(@tmp_path).should be_true
+    File.writable?(@tmp_path).should be_true
+  end
+
+  it "should set and get" do
+    v = rand((2**32)-1)
+    @client.get('test_set_and_get_one_entry').should be_nil
+    @client.set('test_set_and_get_one_entry', v)
+    @client.get('test_set_and_get_one_entry').should eql(v)
+  end
+  
+  
+  it "should expire entries" do
+    v = rand((2**32)-1)
+    @client.get('test_set_with_expiry').should be_nil
+    now = Time.now.to_i
+    @client.set('test_set_with_expiry', v + 2, now)
+    @client.set('test_set_with_expiry', v)
+    sleep(1.0)
+    @client.get('test_set_with_expiry').should eql(v)
+  end
+
+  it "should have age stat" do
+    now = Time.now.to_i
+    @client.set('test_age', 'nibbler')
+    sleep(1.0)
+    @client.get('test_age').should eql('nibbler')
+
+    stats = @client.stats['127.0.0.1:22133']
+    stats.has_key?('queue_test_age_age').should be_true
+    (stats['queue_test_age_age'] >= 1000).should be_true
+  end
+  
+  it "should rotate log" do
+    log_rotation_path = File.join(@tmp_path, 'test_log_rotation')
+
+    Dir.glob("#{log_rotation_path}*").each do |file|
+      File.unlink(file) rescue nil
+    end
+    @client.get('test_log_rotation').should be_nil
+
+    v = 'x' * 8192
+
+    @client.set('test_log_rotation', v)
+    File.size(log_rotation_path).should eql(8207)
+    @client.get('test_log_rotation')
+
+    @client.get('test_log_rotation').should be_nil
+
+    @client.set('test_log_rotation', v)
+    @client.get('test_log_rotation').should eql(v)
+
+    File.size(log_rotation_path).should eql(1)
+    # rotated log should be erased after a successful roll.
+    Dir.glob("#{log_rotation_path}*").size.should eql(1)
+  end
+
+  it "should output statistics per server" do
+    stats = @client.stats
+    assert_kind_of Hash, stats
+    assert stats.has_key?('127.0.0.1:22133')
+    
+    server_stats = stats['127.0.0.1:22133']
+    
+    basic_stats = %w( bytes pid time limit_maxbytes cmd_get version
+                      bytes_written cmd_set get_misses total_connections
+                      curr_connections curr_items uptime get_hits total_items
+                      rusage_system rusage_user bytes_read )
+
+    basic_stats.each do |stat|
+      server_stats.has_key?(stat).should be_true
+    end
+  end
+
+  it "should return valid response with unkown command" do
+    response = @client.add('blah', 1)
+    response.should eql("CLIENT_ERROR bad command line format\r\n")
+  end
+
+  it "should disconnect and reconnect again" do
+    v = rand(2**32-1)
+    @client.set('test_that_disconnecting_and_reconnecting_works', v)
+    @client.reset
+    @client.get('test_that_disconnecting_and_reconnecting_works').should eql(v)
+  end
+  
+  it "should use epoll on linux" do
+    # this may take a few seconds.
+    # the point is to make sure that we're using epoll on Linux, so we can
+    # handle more than 1024 connections.
+    
+    unless IO::popen("uname").read.chomp == "Linux"
+      raise "(Skipping epoll test: not on Linux)"
+      skip = true
+    end
+    fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i
+    unless fd_limit > 1024
+      raise "(Skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024)"
+      skip = true
+    end
+    
+    unless skip
+      v = rand(2**32 - 1)
+      @client.set('test_epoll', v)
+
+      # we can't open 1024 connections to memcache from within this process,
+      # because we will hit ruby's 1024 fd limit ourselves!
+      pid1 = safely_fork do
+        unused_sockets = []
+        600.times do
+          unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+        end
+        Process.kill("USR1", Process.ppid)
+        sleep 90
+      end
+      pid2 = safely_fork do
+        unused_sockets = []
+        600.times do
+          unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+        end
+        Process.kill("USR1", Process.ppid)
+        sleep 90
+      end
+
+      begin
+        client = MemCache.new('127.0.0.1:22133')
+        client.get('test_epoll').should eql(v)
+      ensure
+        Process.kill("TERM", pid1)
+        Process.kill("TERM", pid2)
+      end
+    end
+  end
+  
+  it "should raise error if queue collection is an invalid path" do
+    invalid_path = nil
+    while invalid_path.nil? || File.exist?(invalid_path)
+      invalid_path = File.join('/', Digest::MD5.hexdigest(rand(2**32-1).to_s)[0,8])
+    end
+
+    lambda {
+      StarlingServer::QueueCollection.new(invalid_path)
+    }.should raise_error(StarlingServer::InaccessibleQueuePath)
+  end
+  
+  after do
+    Process.kill("INT", @server_pid)
+    Process.wait(@server_pid)
+    @client.reset
+    FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
+  end
+end