super_queue 0.0.8 → 0.0.9

data/lib/super_queue.rb

@@ -17,8 +17,7 @@ class SuperQueue
   def initialize(opts)
     check_opts(opts)
     opts[:localize_queue] = true unless opts.has_key? :localized_queue
-    opts[:buffer_size] = 100 unless opts.has_key? :buffer_size
-
+    @buffer_size = opts[:buffer_size] || 100
     @localize_queue = opts[:localize_queue]
     @queue_name = generate_queue_name(opts)
     initialize_sqs(opts)
@@ -27,19 +26,19 @@ class SuperQueue
     @waiting.taint
     self.taint
     @mutex = Mutex.new
-    @in_buffer = SizedQueue.new(opts[:buffer_size])
-    @out_buffer = SizedQueue.new(opts[:buffer_size])
-    @deletion_queue = Queue.new
+    @in_buffer = []
+    @out_buffer = []
+    @deletion_queue = []
     @mock_length = 0 if SuperQueue.mocking?
 
-    @sqs_head_tracker = Thread.new { poll_sqs_head }
-    @sqs_tail_tracker = Thread.new { poll_sqs_tail }
-    @garbage_collector = Thread.new { collect_garbage }
+    @sqs_tracker = Thread.new { poll_sqs }
+    @gc = Thread.new { collect_garbage }
   end
 
   def push(p)
     @mutex.synchronize {
       @in_buffer.push p
+      clear_in_buffer if @in_buffer.size >= @buffer_size
       begin
         t = @waiting.shift
         t.wakeup if t
@@ -54,14 +53,14 @@ class SuperQueue
       while true
         if @out_buffer.empty?
           if fill_out_buffer_from_sqs_queue || fill_out_buffer_from_in_buffer
-            return pop_out_buffer(non_block)
+            return pop_out_buffer
           else
             raise ThreadError, "queue empty" if non_block
             @waiting.push Thread.current
             @mutex.sleep
           end
         else
-          return pop_out_buffer(non_block)
+          return pop_out_buffer
         end
       end
     }
@@ -90,21 +89,15 @@ class SuperQueue
   end
 
   def shutdown
-    @sqs_head_tracker.terminate
-    while !@in_buffer.empty? do
-      @sqs.delete_message(q_url, @in_buffer.pop)
-    end  
-    @sqs_tail_tracker.terminate
-    @garbage_collector.terminate
-    while !@deletion_queue.empty?
-      @sqs.delete_message(q_url, @deletion_queue.pop)
-    end
+    @sqs_tracker.terminate
+    @mutex.synchronize { clear_in_buffer }
+    @gc.terminate
+    @mutex.synchronize { clear_deletion_queue }
   end
 
   def destroy
-    @sqs_head_tracker.terminate
-    @sqs_tail_tracker.terminate
-    @garbage_collector.terminate
+    @sqs_tracker.terminate
+    @gc.terminate
     delete_queue
   end
 
@@ -130,6 +123,22 @@ class SuperQueue
 
   private
 
+  def poll_sqs
+    loop do
+      @mutex.synchronize { fill_out_buffer_from_sqs_queue || fill_out_buffer_from_in_buffer } if @out_buffer.empty?
+      @mutex.synchronize { clear_in_buffer } if !@in_buffer.empty? && (@in_buffer.size > @buffer_size)
+      Thread.pass
+    end
+  end
+
+  def collect_garbage
+    loop do
+      #This also needs a condition to clear the del queue if there are any handles where the invisibility is about to expire
+      @mutex.synchronize { clear_deletion_queue } if !@deletion_queue.empty? && (@deletion_queue.size >= (@buffer_size / 2))
+      sleep
+    end
+  end
+
   def check_opts(opts)
     raise "Options can't be nil!" if opts.nil?
     raise "Minimun :buffer_size is 5." if opts[:buffer_size] && (opts[:buffer_size] < 5)
@@ -179,7 +188,8 @@ class SuperQueue
     raise "Couldn't create queue #{queue_name}, or delete existing queue by this name." if q_url.nil?
   end
 
-  def send_message_to_queue(p)
+  def send_message_to_queue
+    p = @in_buffer.shift
     payload = is_a_link?(p) ? p : Base64.encode64(Marshal.dump(p))
     @sqs.send_message(q_url, payload)
     @mock_length += 1 if SuperQueue.mocking?
@@ -248,55 +258,39 @@ class SuperQueue
     orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true  # turn off reverse DNS resolution temporarily
     return "127.0.0.1" if SuperQueue.mocking?
     UDPSocket.open do |s|
-      s.connect '64.233.187.99', 1
+      s.connect '64.233.187 .99', 1
       s.addr.last
     end
   ensure
     Socket.do_not_reverse_lookup = orig
   end
 
-  def poll_sqs_head
-    loop { send_message_to_queue(@in_buffer.pop) }
-  end
-
-  def poll_sqs_tail
-    loop do
-      nil_count = 0
-      while (@out_buffer.size < @out_buffer.max) && (nil_count < 5) # If you get nil 5 times in a row, SQS is probably empty
-        m = get_message_from_queue
-        if m.nil?
-          nil_count += 1
-        else
-          @out_buffer.push m
-          nil_count = 0
-        end
-      end unless sqs_length == 0
-      sleep
-    end
-  end
-
-  def collect_garbage
-    loop { @sqs.delete_message(q_url, @deletion_queue.pop) }
-  end
-
   def fill_out_buffer_from_sqs_queue
-    @sqs_tail_tracker.wakeup if @sqs_tail_tracker.stop?
-    count = 0
-    while @out_buffer.empty? && count != 5
-      sleep 1
-      count += 1
+    return false if sqs_length == 0
+    @gc.wakeup if @gc.stop? # This is the best time to do GC, because there are no pops happening.
+    nil_count = 0
+    while (@out_buffer.size < @buffer_size) && (nil_count < 5) # If you get nil 5 times in a row, SQS is probably empty
+      m = get_message_from_queue
+      if m.nil?
+        nil_count += 1
+      else
+        @out_buffer.push m
+        nil_count = 0
+      end
     end
+    !@out_buffer.empty?
   end
 
   def fill_out_buffer_from_in_buffer
-    while (@out_buffer.size < @out_buffer.max) && !@in_buffer.empty?
-      @out_buffer.push(:payload => @in_buffer.pop)
+    return false if @in_buffer.empty?
+    while (@out_buffer.size <= @buffer_size) && !@in_buffer.empty?
+      @out_buffer.push(:payload => @in_buffer.shift)
     end
     !@out_buffer.empty?
   end
 
-  def pop_out_buffer(non_block)
-    m = @out_buffer.pop(non_block)
+  def pop_out_buffer
+    m = @out_buffer.shift
     @deletion_queue << m[:handle] if m[:handle]
     m[:payload]
   end
@@ -304,4 +298,17 @@ class SuperQueue
   def queue_name
     @queue_name
   end
+
+  def clear_in_buffer
+    while !@in_buffer.empty? do
+      send_message_to_queue
+    end
+  end
+
+  def clear_deletion_queue
+    while !@deletion_queue.empty?
+      @sqs.delete_message(q_url, @deletion_queue.shift)
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: super_queue
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
   prerelease:
 platform: ruby
 authors: