super_queue 0.1.1 → 0.2.1

data/lib/super_queue.rb

@@ -1,4 +1,4 @@
-require 'fog'
+require 'aws-sdk'
 require 'base64'
 require 'socket'
 require 'digest/md5'
@@ -6,20 +6,11 @@ require 'zlib'
 
 class SuperQueue
 
-  def self.mock!
-    @@mock = true
-    Fog.mock!
-  end
-
-  def self.mocking?
-    defined?(@@mock) && @@mock
-  end
-
   def initialize(opts)
+    AWS.eager_autoload! # for thread safety
     check_opts(opts)
-    opts[:localize_queue] = true unless opts.has_key? :localized_queue
+    @should_poll_sqs = opts[:should_poll_sqs]
     @buffer_size = opts[:buffer_size] || 100
-    @localize_queue = opts[:localize_queue]
     @queue_name = generate_queue_name(opts)
     @request_count = 0
     initialize_sqs(opts)
@@ -31,12 +22,11 @@ class SuperQueue
     @in_buffer = []
     @out_buffer = []
     @deletion_queue = []
-    @mock_length = 0 if SuperQueue.mocking?
 
     @compressor = Zlib::Deflate.new
     @decompressor = Zlib::Inflate.new
 
-    @sqs_tracker = Thread.new { poll_sqs }
+    @sqs_tracker = Thread.new { poll_sqs } if @should_poll_sqs
     @gc = Thread.new { collect_garbage }
   end
 
@@ -94,14 +84,14 @@ class SuperQueue
   end
 
   def shutdown
-    @sqs_tracker.terminate
+    @sqs_tracker.terminate if @should_poll_sqs
     @mutex.synchronize { clear_in_buffer }
     @gc.terminate
     @mutex.synchronize { clear_deletion_queue }
   end
 
   def destroy
-    @sqs_tracker.terminate
+    @sqs_tracker.terminate if @should_poll_sqs
     @gc.terminate
     delete_queue
   end
@@ -126,10 +116,6 @@ class SuperQueue
     queue_name
   end
 
-  def localized?
-    !!@localize_queue
-  end
-
   private
 
   #
@@ -148,11 +134,11 @@ class SuperQueue
 
   def create_sqs_connection(opts)
     aws_options = {
-      :aws_access_key_id => opts[:aws_access_key_id], 
-      :aws_secret_access_key => opts[:aws_secret_access_key]
+      :access_key_id => opts[:aws_access_key_id], 
+      :secret_access_key => opts[:aws_secret_access_key]
     }
     begin
-      @sqs = Fog::AWS::SQS.new(aws_options)
+      @sqs = AWS::SQS.new(aws_options)
     rescue Exception => e
       raise e
     end
@@ -161,20 +147,29 @@ class SuperQueue
   def create_sqs_queue(opts)
     retries = 0
     begin
-      @sqs_queue = new_sqs_queue(opts)
+      @sqs_queue = find_queue_by_name || new_sqs_queue(opts)
       check_for_queue_creation_success
     rescue RuntimeError => e
       retries += 1
-      (retries >= 10) ? retry : raise(e)
+      sleep 1
+      (retries >= 20) ? retry : raise(e)
+    end
+  end
+
+  def find_queue_by_name
+    begin
+      @sqs.queues.named(queue_name)
+    rescue AWS::SQS::Errors::NonExistentQueue
+      return nil
     end
   end
 
   def new_sqs_queue(opts)
     @request_count += 1
     if opts[:visibility_timeout]
-      @sqs.create_queue(queue_name, {"DefaultVisibilityTimeout" => opts[:visibility_timeout]})
+      @sqs.queues.create(queue_name, { :visibility_timeout => opts[:visibility_timeout] })
     else
-      @sqs.create_queue(queue_name)
+      @sqs.queues.create(queue_name)
     end
   end
 
@@ -187,56 +182,59 @@ class SuperQueue
     raise "Couldn't create queue #{queue_name}, or delete existing queue by this name." if q_url.nil?
   end
 
-  def send_message_to_queue
-    p = @in_buffer.shift
-    payload = is_a_link?(p) ? p : encode(p)
-    retries = 0
-    begin
+  def send_messages_to_queue
+    number_of_batches = @in_buffer.size / 10
+    number_of_batches += 1 if @in_buffer.size % 10
+    batches = []
+    number_of_batches.times do
+      batch = []
+      10.times do
+        p = @in_buffer.shift
+        obj = is_a_link?(p) ? p : encode(p)
+        batch << obj
+      end
+      batches << batch
+    end
+
+    #Ugliness! But I'm not sure how else to tackle this at the moment
+    batches.each do |b|
       @request_count += 1
-      @sqs.send_message(q_url, payload)
-    rescue Excon::Errors::BadRequest => e
-      retries += 1
-      (retries >= 10) ? retry : raise(e)
+      @sqs_queue.batch_send(b)
     end
-    @mock_length += 1 if SuperQueue.mocking?
   end
 
-  def get_message_from_queue
-    message = @sqs.receive_message(q_url)
-    return nil if  message.body.nil? || message.body['Message'].first.nil?
-    handle = message.body['Message'].first['ReceiptHandle']
-    ser_obj = message.body['Message'].first['Body']
-    return nil if ser_obj.nil? || ser_obj.empty?
-    @mock_length -= 1 if SuperQueue.mocking?
-    @request_count += 1
-    return {:handle => handle, :payload => ser_obj} if is_a_link?(ser_obj)
-    { :handle => handle, :payload => decode(ser_obj) }
+  def get_messages_from_queue(number_of_messages_to_receive)
+    messages = []
+    number_of_batches = number_of_messages_to_receive / 10
+    number_of_batches += 1 if number_of_messages_to_receive % 10
+    number_of_batches.times do |c|
+      batch = @sqs_queue.receive_messages(:limit => 10)
+      batch.each do |m|
+        if is_a_link?(m.body)
+          obj = {:message => m, :payload => m.body}
+        else
+          obj = { :message => m, :payload => decode(m.body) }
+        end
+        messages << obj
+      end
+      @request_count += 1
+    end
+    messages
   end
 
   def sqs_length
-    return @mock_length if SuperQueue.mocking?
-    body = @sqs.get_queue_attributes(q_url, "ApproximateNumberOfMessages").body
-    @request_count += 1
-    begin
-      retval = 0
-      if body
-        attrs = body["Attributes"]
-        if attrs
-          retval = attrs["ApproximateNumberOfMessages"]
-        end
-      end
-    end until !retval.nil?
-    retval
+    n = @sqs_queue.approximate_number_of_messages
+    return n.is_a?(Integer) ? n : 0
   end
 
   def delete_queue
     @request_count += 1
-    @sqs.delete_queue(q_url)
+    @sqs_queue.delete
   end
 
   def clear_deletion_queue
     while !@deletion_queue.empty?
-      @sqs.delete_message(q_url, @deletion_queue.shift)
+      @sqs_queue.batch_delete(@deletion_queue[0..9])
       @request_count += 1
     end
   end
@@ -247,15 +245,9 @@ class SuperQueue
   def fill_out_buffer_from_sqs_queue
     return false if sqs_length == 0
     @gc.wakeup if @gc.stop? # This is the best time to do GC, because there are no pops happening.
-    nil_count = 0
-    while (@out_buffer.size < @buffer_size) && (nil_count < 5) # If you get nil 5 times in a row, SQS is probably empty
-      m = get_message_from_queue
-      if m.nil?
-        nil_count += 1
-      else
-        @out_buffer.push m
-        nil_count = 0
-      end
+    while (@out_buffer.size < @buffer_size)
+      messages = get_messages_from_queue(@buffer_size - @out_buffer.size)
+      messages.each { |m| @out_buffer.push m }
     end
     !@out_buffer.empty?
   end
@@ -270,13 +262,13 @@ class SuperQueue
 
   def pop_out_buffer
     m = @out_buffer.shift
-    @deletion_queue << m[:handle] if m[:handle]
+    @deletion_queue << m[:message] if m[:message]
     m[:payload]
   end
 
   def clear_in_buffer
     while !@in_buffer.empty? do
-      send_message_to_queue
+      send_messages_to_queue
     end
   end
 
@@ -291,24 +283,16 @@ class SuperQueue
   # Misc helper methods
   #
   def encode(p)
-    text = Base64.encode64(Marshal.dump(p))
-    retval = nil
-    retries = 0
-    begin
-      retval = @compressor.deflate(text)
-      retries += 1
-    end until !(retval.nil? || retval.empty?) || (retries > 5)
-    retval
+    Base64.urlsafe_encode64(Marshal.dump(p))
   end
 
   def decode(ser_obj)
-    text = nil
-    retries = 0
-    begin
-      text = @decompressor.inflate(ser_obj)
-      retries += 1
-    end until !(text.nil? || text.empty?) || (retries > 5)
-    Marshal.load(Base64.decode64(text))
+    #puts "Decode: Decoding message #{ser_obj} from base64..."
+    obj = Base64.urlsafe_decode64(ser_obj)
+    #puts "Decode: Object decoded as #{obj}. Doing Marshal load..."
+    ret = Marshal.load(obj)
+    #puts "Decode: Marshal loaded as #{ret}"
+    ret
   end
 
   def is_a_link?(s)
@@ -318,15 +302,7 @@ class SuperQueue
 
   def generate_queue_name(opts)
     q_name = opts[:name] || random_name
-    if opts[:namespace] && opts[:localize_queue]
-      "#{@namespace}-#{Digest::MD5.hexdigest(local_ip)}-#{q_name}"
-    elsif opts[:namespace]
-      "#{@namespace}-#{q_name}"
-    elsif opts[:localize_queue]
-      "#{Digest::MD5.hexdigest(local_ip)}-#{q_name}"
-    else
-      q_name
-    end
+    return opts[:namespace] ? "#{@namespace}-#{q_name}" : q_name
   end
 
   #
@@ -334,7 +310,7 @@ class SuperQueue
   #
   def q_url
     return @q_url if @q_url
-    @q_url = @sqs_queue.body['QueueUrl']
+    @q_url = @sqs_queue.url
     @q_url
   end
 
@@ -347,17 +323,6 @@ class SuperQueue
     @queue_name
   end
 
-  def local_ip
-    orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true  # turn off reverse DNS resolution temporarily
-    return "127.0.0.1" if SuperQueue.mocking?
-    UDPSocket.open do |s|
-      s.connect '64.233.187 .99', 1
-      s.addr.last
-    end
-  ensure
-    Socket.do_not_reverse_lookup = orig
-  end
-
   #
   # Maintence thread-related methods
   #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: super_queue
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
   prerelease:
 platform: ruby
 authors:
@@ -12,18 +12,18 @@ cert_chain: []
 date: 2013-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fog
+  name: aws-sdk
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.6.5
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.6.5
     none: false
   prerelease: false
   type: :runtime