queuel 0.3.1 → 0.4.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    queuel (0.3.1)
+    queuel (0.4.0)
       mono_logger
       multi_json
       thread

data/README.md

@@ -7,6 +7,7 @@
 Queuel is a kewl, lite wrapper around Queue interfaces. Currently it implements:
 
 * IronMQ
+* Amazon SQS
 * Null pattern
 
 Each of these should reliably implement:
@@ -89,6 +90,11 @@ Queuel.receive break_if_nil: true do |message|
 end
 ```
 
+Notes specific to the SQS engine:
+A config should be added to provide `bucket_name`, `max_bytesize`, `access_key`
+and `secret_access_key`. Without these, messages over the max_bytesize setting
+(defaults to 64kb) will be dropped from the queue.
+
 #### Caveats of the receiver
 
 * Your block must return true in order to not replace the message to the Queue
@@ -105,7 +111,7 @@ message.delete    # => Delete the message
 #### Parsing
 
 Queuel uses [MultiJson](https://github.com/intridea/multi_json) to provide
-some auto-message decoding/encodeing features. With MultiJson you may install your own engine
+some auto-message decoding/encoding features. With MultiJson you may install your own engine
 (like [Oj](https://github.com/ohler55/oj)).
 
 Because of the parsing given, you will default to encoding and decoding JSON:

data/config/s3_example.yml

@@ -0,0 +1,9 @@
+staging: &staging
+  s3:
+    access_key: "[access_key]"
+    secret_access_key: "[secret_key]"
+    bucket_name: "example-bucket-name"
+    max_bytesize: 100000
+
+development:
+  <<: *staging

data/lib/queuel/base/engine.rb

@@ -12,6 +12,7 @@ module Queuel
 
       def initialize(credentials = {})
         self.credentials = credentials
+        self.bucket_name = credentials[:bucket_name]
         self.memoized_queues = {}
       end
 
@@ -21,6 +22,7 @@ module Queuel
 
       private
       attr_accessor :credentials
+      attr_accessor :bucket_name
       attr_accessor :memoized_queues
 
       def client

data/lib/queuel/base/queue.rb

@@ -32,6 +32,10 @@ module Queuel
         poller_klass.new(self, block, options, thread_count).poll
       end
 
+      def size
+        raise NotImplementedError, "must implement #size"
+      end
+
       private
       attr_accessor :client
       attr_accessor :name

data/lib/queuel/iron_mq/queue.rb

@@ -17,6 +17,10 @@ module Queuel
         queue_connection.post build_push_message(message, options)
       end
 
+      def size
+        queue_connection.size
+      end
+
       private
       def pop_bare_message(options = {})
         queue_connection.get options.merge(default_get_message_options)

data/lib/queuel/sqs/engine.rb

@@ -7,8 +7,13 @@ module Queuel
 
       AWSSDKMissingError = Class.new(StandardError)
 
+      def queue(which_queue)
+        memoized_queues[which_queue.to_s] ||= queue_klass.new(client, which_queue, credentials)
+      end
+
       private
 
+
       def client_klass
         if defined?(::AWS::SQS)
           ::AWS::SQS

data/lib/queuel/sqs/message.rb

@@ -1,10 +1,69 @@
 module Queuel
   module SQS
     class Message < Base::Message
+      # if message_object exists (not nil), receive the data, otherwise push
       def raw_body
-        @raw_body ||
-          (message_object && raw_body_with_sns_check) ||
-          encoded_body
+        @raw_body ||= message_object.nil? ? push_message : pull_message
+      end
+
+      def generate_key
+        key = [
+          (Time.now.to_f * 10000).to_i,
+          SecureRandom.urlsafe_base64,
+          Thread.current.object_id
+        ].join('-')
+        key
+      end
+
+      def push_message
+        if encoded_body.bytesize > max_bytesize
+          key = generate_key
+          write_to_s3(encoded_body, key)
+          self.body = { 'queuel_s3_object' => key }
+        end
+        encoded_body
+      end
+
+      def pull_message
+        begin
+          decoded_body = JSON.parse(message_object.body)
+          if decoded_body.key?('queuel_s3_object')
+            read_from_s3 decoded_body[:queuel_s3_object]
+          else
+            message_object.body
+          end
+        rescue ::JSON::ParserError, TypeError
+          raw_body_with_sns_check
+        end
+      end
+
+      def max_bytesize
+        options[:max_bytesize] || 64 * 1024
+      end
+
+      def self.s3
+        @s3 ||= AWS::S3.new(
+                  :access_key_id => options[:access_token],
+                  :secret_access_key => options[:secret_access_token] )
+      end
+
+      def read_from_s3 key
+        object = s3.buckets[options[:bucket_name]].objects[key]
+        object.read
+      end
+
+      # this assumes you've set up some rules for file expiration on your
+      # configured bucket.
+      def write_to_s3(message, key)
+        begin
+          my_bucket = s3.buckets[options[:bucket_name]]
+          my_bucket.objects[key].write(message)
+        rescue
+          raise BucketDoesNotExistError, "Bucket has either expired or does not exist"
+        end
+      end
+
+      class BucketDoesNotExistError < StandardError
       end
 
       def delete

data/lib/queuel/sqs/queue.rb

@@ -5,6 +5,15 @@ module Queuel
     class Queue < Base::Queue
       extend Forwardable
 
+      attr_accessor :credentials
+
+      def initialize(client, queue_name, credentials={})
+        self.client = client
+        self.name = queue_name
+        self.credentials = credentials
+      end
+
+
       def push(message, options = {})
         queue_connection.send_message build_push_message(message, options)
       end
@@ -13,7 +22,18 @@ module Queuel
         queue_connection.approximate_number_of_messages
       end
 
+      def size
+        approximate_number_of_messages
+      end
+
       private
+
+
+      def build_new_message(bare_message, options = {})
+        message_klass.new(bare_message, credentials)
+      end
+
+
       def pop_bare_message(options = {})
         queue_connection.receive_message options
       end

data/lib/queuel/version.rb

@@ -1,3 +1,3 @@
 module Queuel
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/spec/lib/queuel/base/queue_spec.rb

@@ -8,7 +8,7 @@ module Queuel
 
       its(:thread_count) { should == 1 }
 
-      it "fails on non-impleneted push" do
+      it "fails on non-implemented push" do
         expect { subject.push("") }.to raise_error NotImplementedError
       end
 

data/spec/lib/queuel/iron_mq/queue_spec.rb

@@ -2,9 +2,37 @@ require 'spec_helper'
 module Queuel
   module IronMq
     describe Queue do
+      let(:message) { double "Message", body: "uhuh" }
+      let(:client) { double "ClientObject" }
+      let(:name) { "venues queue" }
       let(:queue_object_with_message) { double "QueueObject", get: message, peek: [message] }
       let(:queue_object_with_nil_message) { double "QueueObject", get: nil, peek: nil }
+
+      subject do
+        described_class.new client, name
+      end
+
       it_should_behave_like "a queue"
+
+      describe "size" do
+        it "should check the queue_connection for size" do
+          client.stub queue: queue_object_with_message
+          queue_object_with_message.should_receive(:size)
+          subject.size
+        end
+      end
+
+      describe "peek" do
+        before do
+          not_for_null do
+            client.stub queue: queue_object_with_message
+          end
+        end
+
+        it "should take options and return an array" do
+          subject.peek(option: true).should be_an Array
+        end
+      end
     end
   end
 end

data/spec/lib/queuel/sqs/message_spec.rb

@@ -13,12 +13,33 @@ module Queuel
         before do
           subject.stub decode_body?: false
           message_object.stub(:as_sns_message).and_raise ::JSON::ParserError
+          Queuel.configure { engine :sqs }
         end
 
         its(:id) { should == 1 }
         its(:body) { should == "body" }
         its(:queue) { should == queue_double }
 
+        describe "when pulling an oversized message" do
+          let(:body) { '{"queuel_s3_object": "whatever" }' }
+
+          it "should call read_from_s3" do
+            subject.should_receive(:read_from_s3)
+            subject.raw_body
+          end
+        end
+
+        describe "when pushing an oversized json hash" do
+          before do
+            subject.send("message_object=", nil)
+            subject.stub(:encoded_body).and_return double("body", bytesize: subject.max_bytesize+1)
+          end
+          it "should call write_to_s3" do
+            subject.should_receive(:write_to_s3)
+            subject.raw_body
+          end
+        end
+
         describe "with json" do
           let(:body) { '{"username":"jon"}' }
           before do
@@ -30,14 +51,14 @@ module Queuel
         end
 
         describe "with valid SNS message" do
-          let(:sns_body) { "Hello From SNS"}
+          let(:sns_body) { "Hello From SNS" }
           before do
             message_object.stub(:as_sns_message).and_return double("SNSMessage", body: sns_body)
           end
           its(:raw_body) { should == sns_body }
           its(:raw_body) { should_not == message_object.body}
 
-          describe "that is json" do
+          describe "which is json" do
             let(:sns_body) { '{"username":"jon"}' }
             before do
               subject.stub decode_body?: true

data/spec/lib/queuel/sqs/queue_spec.rb

@@ -1,11 +1,31 @@
 require 'spec_helper'
 module Queuel
-  module IronMq
+  module SQS
     describe Queue do
-      let(:queue_object_with_message) { double "QueueObject", get: message, peek: [message] }
-      let(:queue_object_with_nil_message) { double "QueueObject", get: nil, peek: nil }
+      let(:message) { double "Message", body: "uhuh" }
+      let(:client) { double "ClientObject" }
+      let(:name) { "venues queue" }
+      let(:credentials) {{ access_key: "none", secret_access_key: "none" }}
+      let(:queue_object_with_message) { double "QueueObject", get: message, receive_message: message }
+      let(:queue_object_with_nil_message) { double "QueueObject", get: nil, receive_message: nil }
+
+      subject do
+        described_class.new client, name, credentials
+      end
+
+      before do
+        message.stub_chain :as_sns_message, body: "uhuh"
+        client.stub_chain :queues, named: queue_object_with_message
+      end
 
       it_should_behave_like "a queue"
+
+      describe "size" do
+        it "should check the queue_connection's approximate_number_of_messages for size" do
+          queue_object_with_message.should_receive :approximate_number_of_messages
+          subject.size
+        end
+      end
     end
   end
 end

data/spec/support/queue_shared_example.rb

@@ -7,29 +7,16 @@ shared_examples "a queue" do
   end
 
   # Poller object handles this
-  it { should respond_to :receive }
   it { should respond_to :push }
   it { should respond_to :pop }
-  it { should respond_to :peek }
-
-  describe "peek" do
-    before do
-      not_for_null do
-        client.stub queue: queue_object_with_message
-      end
-    end
-
-    it "should take options and return an array" do
-      subject.peek(option: true).should be_an Array
-    end
-  end
+  it { should respond_to :size }
 
   describe "pop" do
     describe "with messages" do
       before do
         not_for_null do
           client.stub queue: queue_object_with_message
-          client.stub named: queue_object_with_message
+          client.stub_chain :queues, named: queue_object_with_message
         end
       end
 
@@ -52,7 +39,7 @@ shared_examples "a queue" do
     before do
       not_for_null do
         client.stub queue: queue_object_with_nil_message
-        client.stub named: queue_object_with_message
+        client.stub_chain :queues, named: queue_object_with_nil_message
       end
     end
 

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: queuel
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
+  prerelease: 
 platform: ruby
 authors:
 - Jon Phenow
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-18 00:00:00.000000000 Z
+date: 2014-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -55,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,6 +70,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -69,6 +78,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: iron_mq
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -76,6 +86,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -83,6 +94,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -90,6 +102,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -97,6 +110,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -104,6 +118,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -111,6 +126,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: guard-bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -118,6 +134,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -125,6 +142,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -132,6 +150,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -139,6 +158,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: guard-cane
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -146,6 +166,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -153,6 +174,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-inotify
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -160,6 +182,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -167,6 +190,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-fsevent
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -174,6 +198,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -181,6 +206,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-fchange
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -188,6 +214,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -195,6 +222,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: cane
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -202,6 +230,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -209,6 +238,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -216,6 +246,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -223,6 +254,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -230,6 +262,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -237,6 +270,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -244,6 +278,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -251,6 +286,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: mono_logger
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -258,6 +294,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -265,6 +302,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -272,6 +310,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -297,6 +336,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- config/s3_example.yml
 - lib/queuel.rb
 - lib/queuel/base/engine.rb
 - lib/queuel/base/message.rb
@@ -351,26 +391,33 @@ files:
 homepage: ''
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2472164195127432886
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2472164195127432886
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 1.8.23.2
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Light Queue wrapper tool
 test_files:
 - spec/lib/queuel/base/queue_spec.rb

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MTYyYjg5NTFlNjA2ZjNkYjFlMWExNmM1MWE5OTAzYzRjZjFlZDU0ZQ==
-  data.tar.gz: !binary |-
-    MGNhN2Q4YzQ2ZGUzZjg4MGFjMDdhYTVlZDlkZjYwYzgwNjE1ZDI0MQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    MTFiMzQ3NjYzZDU3OGQxMTI0MzU5MWY2YTM5ODRmNzQ1MDE0ZDM0N2U0Njk1
-    MWNjY2Q2Mjg0NmIyYTc2MWVjOTkxNzJlZjZlYWMyMGRmOGYyNjMxZmExN2U3
-    NzAyYTNlZWY3NjFkYjVkOGU4Yzc3OWE1YmJiYjhmMmI5OGRlNjk=
-  data.tar.gz: !binary |-
-    ODM1OTBlYzhkMjNmMjE2MGZlYzE5MmU1ZWQxNDRjM2FlMmY1MzJkYTUwYmEz
-    ZTVlMWE4MTdmYzdhZGYyMzI2Y2EzNzcwZjcyNjU0NzEyOGI4NGNlMjIyMzNl
-    YWIyYmVjMGY3NDRiYjdkZmY4NDEwNmU2YTJiNDllMmQyMDllOWU=