queuel 0.4.1 → 0.4.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    queuel (0.4.1)
+    queuel (0.4.2)
       mono_logger
       multi_json
       thread

data/README.md

@@ -1,10 +1,12 @@
-# Queuel
+Queuel
+======
+
 [![Gem Version](https://badge.fury.io/rb/queuel.png)](http://badge.fury.io/rb/queuel)
 [![Build Status](https://travis-ci.org/sportngin/queuel.png?branch=master)](https://travis-ci.org/sportngin/queuel)
 [![Code Climate](https://codeclimate.com/github/sportngin/queuel.png)](https://codeclimate.com/github/sportngin/queuel)
 [![Coverage Status](https://coveralls.io/repos/sportngin/queuel/badge.png)](https://coveralls.io/r/sportngin/queuel)
 
-Queuel is a kewl, lite wrapper around Queue interfaces. Currently it implements:
+Queuel is a 'kewl', lite wrapper around Queue interfaces. Currently it implements:
 
 * IronMQ
 * Amazon SQS
@@ -18,7 +20,10 @@ Each of these should reliably implement:
 
 Along with some further conveniences.
 
-## Installation
+
+
+Installation
+------------
 
 Add this line to your application's Gemfile as well as the proper Gem for
 your queuing:
@@ -31,7 +36,9 @@ gem 'queuel'
 
 And then execute:
 
-    $ bundle
+```bash
+$ bundle
+```
 
 You will then want to configure:
 
@@ -43,7 +50,7 @@ Queuel.configure do
   # requirement depends on your Queue
   credentials token: 'asdufasdf8a7sd8fa7sdf', project_id: 'project_id'
 
-  # currently only [:iron_mq, :null] available
+  # currently [:iron_mq, :sqs, :null] available
   engine :iron_mq
 
   # For Queuel.recevier {} you can configure more than one thread to
@@ -65,7 +72,10 @@ Queuel.configure do
 end
 ```
 
-## Usage
+
+
+Usage
+-----
 
 ### General Queue API
 
@@ -90,15 +100,25 @@ Queuel.receive break_if_nil: true do |message|
 end
 ```
 
-Notes specific to the SQS engine:
-A config should be added to provide `bucket_name`, `max_bytesize`, `access_key`
-and `secret_access_key`. Without these, messages over the max_bytesize setting
-(defaults to 64kb) will be dropped from the queue.
-
 #### Caveats of the receiver
 
 * Your block must return true in order to not replace the message to the Queue
 
+#### SQS s3 fallback
+
+Currently the SQS engine is the only engine with the s3 fallback support and
+takes the following keys:
+
+* `s3_access_key_id`
+* `s3_secret_access_key`
+* `s3_bucket_name`
+* `max_bytesize` (optional)
+
+With these in place, messages over the `max_bytesize` (defaults to 64kb) will
+be sent to the designated bucket.  Without this in place, messages over SQS's
+limit be dropped from the queue.
+
+
 ### The message
 
 ```ruby
@@ -138,7 +158,9 @@ Queuel.receive decode: false
 ```
 
 
-## Contributing
+
+Contributing
+------------
 
 1. Fork it
 2. Create your feature branch (`git checkout -b my-new-feature`)

data/lib/queuel/sqs/message.rb

@@ -1,80 +1,94 @@
 module Queuel
   module SQS
     class Message < Base::Message
-      # if message_object exists (not nil), receive the data, otherwise push
+
       def raw_body
-        @raw_body ||= message_object.nil? ? push_message : pull_message
+        @raw_body ||= message_object ? pull_message : push_message
       end
 
-      def generate_key
-        key = [
-          (Time.now.to_f * 10000).to_i,
-          SecureRandom.urlsafe_base64,
-          Thread.current.object_id
-        ].join('-')
-        key
+      def delete
+        message_object.delete
+      end
+
+      [:id, :queue].each do |delegate|
+        define_method(delegate) do
+          instance_variable_get("@#{delegate}") || message_object && message_object.public_send(delegate)
+        end
       end
 
       def push_message
         if encoded_body.bytesize > max_bytesize
           key = generate_key
-          write_to_s3(encoded_body, key)
+          s3_transaction(:write, key, encoded_body)
           self.body = { 'queuel_s3_object' => key }
         end
         encoded_body
       end
+      private :push_message
 
       def pull_message
         begin
-          decoded_body = JSON.parse(message_object.body)
-          if decoded_body.key?('queuel_s3_object')
-            read_from_s3 decoded_body[:queuel_s3_object]
+          decoded_body = decoder.call(message_object.body)
+          if decoded_body.key?(:queuel_s3_object)
+            s3_transaction(:read, decoded_body[:queuel_s3_object])
           else
             message_object.body
           end
-        rescue ::JSON::ParserError, TypeError
+        rescue Queuel::Serialization::Json::SerializationError, TypeError
           raw_body_with_sns_check
         end
       end
+      private :pull_message
 
       def max_bytesize
         options[:max_bytesize] || 64 * 1024
       end
+      private :max_bytesize
 
-      def self.s3
-        @s3 ||= AWS::S3.new(
-                  :access_key_id => options[:access_token],
-                  :secret_access_key => options[:secret_access_token] )
-      end
-
-      def read_from_s3 key
-        object = s3.buckets[options[:bucket_name]].objects[key]
-        object.read
+      def s3
+        @s3 ||= ::AWS::S3.new(
+                  :access_key_id => options[:s3_access_key_id],
+                  :secret_access_key => options[:s3_secret_access_key] )
       end
+      private :s3
 
-      # this assumes you've set up some rules for file expiration on your
-      # configured bucket.
-      def write_to_s3(message, key)
-        begin
-          my_bucket = s3.buckets[options[:bucket_name]]
-          my_bucket.objects[key].write(message)
-        rescue
+      # @method - write or read
+      # @args - key and message if writing
+      def s3_transaction(method, *args)
+        bucket_name = options[:s3_bucket_name]
+        raise NoBucketNameSupplied if bucket_name.nil?
+        my_bucket = s3.buckets[bucket_name]
+        if my_bucket.exists?
+          begin
+            send("s3_#{method}", my_bucket, *args)
+          rescue AWS::S3::Errors::AccessDenied => e
+            raise InsufficientPermissions, "Unable to read from bucket: #{e.message}"
+          end
+        else
           raise BucketDoesNotExistError, "Bucket has either expired or does not exist"
         end
       end
+      private :s3_transaction
 
-      class BucketDoesNotExistError < StandardError
+      def s3_read(bucket, *args)
+        bucket.objects[args[0]].read
       end
+      private :s3_read
 
-      def delete
-        message_object.delete
+      def s3_write(bucket, *args)
+        bucket.objects[args[0]].write(args[1])
       end
+      private :s3_write
 
-      [:id, :queue].each do |delegate|
-        define_method(delegate) do
-          instance_variable_get("@#{delegate}") || message_object && message_object.public_send(delegate)
-        end
+      def generate_key
+        key = [
+          (Time.now.to_f * 10000).to_i,
+          SecureRandom.urlsafe_base64,
+          Thread.current.object_id
+        ].join('-')
+        key
       end
+      private :generate_key
 
       def raw_body_with_sns_check
         begin
@@ -84,6 +98,11 @@ module Queuel
         end
       end
       private :raw_body_with_sns_check
+
+
+      class NoBucketNameSupplied < Exception; end
+      class InsufficientPermissions < StandardError; end
+      class BucketDoesNotExistError < StandardError; end
     end
   end
 end

data/lib/queuel/sqs/queue.rb

@@ -15,7 +15,8 @@ module Queuel
 
 
       def push(message, options = {})
-        queue_connection.send_message build_push_message(message, options)
+        built_message = build_push_message message, options.merge(credentials)
+        queue_connection.send_message built_message
       end
 
       def approximate_number_of_messages
@@ -28,17 +29,14 @@ module Queuel
 
       private
 
-
       def build_new_message(bare_message, options = {})
         message_klass.new(bare_message, credentials)
       end
 
-
       def pop_bare_message(options = {})
         queue_connection.receive_message options
       end
 
-
       def queue_connection
         @queue_connection ||= client.queues.named(name)
       end

data/lib/queuel/version.rb

@@ -1,3 +1,3 @@
 module Queuel
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

data/spec/lib/queuel/sqs/message_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'json'
+require 'aws-sdk'
 module Queuel
   module SQS
     describe Message do
@@ -20,26 +21,48 @@ module Queuel
         its(:body) { should == "body" }
         its(:queue) { should == queue_double }
 
-        describe "when pulling an oversized message" do
-          let(:body) { '{"queuel_s3_object": "whatever" }' }
+        it "calls raw_body_with_sns_check if not a json object" do
+          subject.should_receive(:raw_body_with_sns_check)
+          subject.raw_body
+        end
+
+        context "when pulling an oversized message" do
+          let(:body) { '{"queuel_s3_object": "whatever"}' }
+          let(:message_object) { double "SQSMessage", id: 2, body: body, queue: queue_double }
+          subject { described_class.new(message_object) }
 
-          it "should call read_from_s3" do
-            subject.should_receive(:read_from_s3)
+          it "calls s3_transaction with read" do
+            subject.should_receive(:s3_transaction).with(:read, "whatever")
             subject.raw_body
           end
         end
 
-        describe "when pushing an oversized json hash" do
+        context "when pushing an oversized json hash" do
+          let(:message) { double("body", bytesize: subject.send(:max_bytesize) + 1) }
           before do
             subject.send("message_object=", nil)
-            subject.stub(:encoded_body).and_return double("body", bytesize: subject.max_bytesize+1)
+            subject.stub(:encoded_body).and_return message
           end
-          it "should call write_to_s3" do
-            subject.should_receive(:write_to_s3)
+
+          it "should call s3_transaction with write" do
+            subject.stub(:generate_key).and_return "key"
+            subject.should_receive(:s3_transaction).with(:write, "key", message)
             subject.raw_body
           end
         end
 
+        describe "#s3" do
+          subject do
+            described_class.new message_object, :s3_access_key_id => "stuff",
+                                                :s3_secret_access_key => "derp"
+          end
+
+          it "sets the s3 object" do
+            AWS::S3.should_receive(:new).with(:access_key_id => "stuff", :secret_access_key => "derp")
+            subject.send :s3
+          end
+        end
+
         describe "with json" do
           let(:body) { '{"username":"jon"}' }
           before do

data/spec/lib/queuel/sqs/queue_spec.rb

@@ -26,6 +26,29 @@ module Queuel
           subject.size
         end
       end
+
+      describe "push" do
+        before do
+          queue_object_with_message.should_receive(:send_message)
+                                   .with('foobar')
+        end
+
+        it "receives a call to build message with the credentials" do
+          subject.should_receive(:build_push_message)
+                 .with("foobar", credentials)
+                 .and_return('foobar')
+
+          subject.push "foobar"
+        end
+
+        it "merges options that are passed in" do
+          subject.should_receive(:build_push_message)
+                 .with("foobar", {:foo => 'bar'}.merge(credentials))
+                 .and_return('foobar')
+
+          subject.push "foobar", :foo => 'bar'
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queuel
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-08 00:00:00.000000000 Z
+date: 2014-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -403,7 +403,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3766388311581194323
+      hash: 2118927277156940265
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -412,7 +412,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3766388311581194323
+      hash: 2118927277156940265
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23.2