carnivore-sqs 0.1.0

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/README.md

@@ -0,0 +1,24 @@
+# Carnivore SQS
+
+Provides SQS `Carnivore::Source`
+
+# Usage
+
+```ruby
+require 'carnivore'
+require 'carnivore-sqs'
+
+Carnivore.configure do
+  source = Carnivore::Source.build(
+    :type => {
+      :fog => {...},
+      :queues => ['arn:aws:sqs:...']
+    }
+  )
+end.start!
+```
+
+# Info
+* Carnivore: https://github.com/heavywater/carnivore
+* Repository: https://github.com/heavywater/carnivore-sqs
+* IRC: Freenode @ #heavywater

data/carnivore-sqs.gemspec

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'carnivore-sqs/version'
+Gem::Specification.new do |s|
+  s.name = 'carnivore-sqs'
+  s.version = Carnivore::Sqs::VERSION.version
+  s.summary = 'Message processing helper'
+  s.author = 'Chris Roberts'
+  s.email = 'chrisroberts.code@gmail.com'
+  s.homepage = 'https://github.com/heavywater/carnivore-sqs'
+  s.description = 'Carnivore SQS source'
+  s.require_path = 'lib'
+  s.add_dependency 'carnivore', '>= 0.1.8'
+  s.add_dependency 'fog'
+  s.files = Dir['**/*']
+end

data/lib/carnivore-sqs/sqs.rb

@@ -0,0 +1,135 @@
+require 'fog'
+require 'carnivore/source'
+
+module Carnivore
+  class Source
+    class Sqs < Source
+
+      OUTPUT_REPEAT_EVERY=20
+
+      attr_reader :pause_time
+
+      def setup(args={})
+        @fog = nil
+        @connection_args = args[:fog]
+        case args[:queues]
+        when Hash
+          @queues = args[:queues]
+        else
+          @queues = Array(args[:queues]).flatten.compact
+          @queues = Hash[*(
+              @queues.size.times.map(&:to_i).zip(@queues).flatten
+          )]
+        end
+        @queues.values.map do |q|
+          q.replace(format_queue(q))
+        end
+        if(args[:processable_queues])
+          @processable_queues = Array(args[:processable_queues]).flatten.compact
+        end
+        @pause_time = args[:pause] || 5
+        @receive_timeout = after(args[:receive_timeout] || 30){ terminate }
+        debug "Creating SQS source instance <#{name}>"
+        debug "Handled queues: #{@queues.inspect}"
+      end
+
+      def format_queue(q)
+        q.include?('.com') ? q : "/#{q.split(':')[-2,2].join('/')}"
+      end
+
+      def connect
+        @fog = Fog::AWS::SQS.new(@connection_args)
+      end
+
+      def receive(n=1)
+        count = 0
+        msgs = []
+        while(msgs.empty?)
+          msgs = []
+          @receive_timeout.reset
+          msgs = queues.map do |q|
+            m = @fog.receive_message(q, 'MaxNumberOfMessages' => n).body['Message']
+            m.map! do |msg|
+              msg.merge('SourceQueue' => q)
+            end
+          end.flatten.compact
+          @receive_timeout.reset
+          if(msgs.empty?)
+            if(count == 0)
+              debug "Source<#{name}> no message received. Sleeping for #{pause_time} seconds."
+            elsif(count % OUTPUT_REPEAT_EVERY == 0)
+              debug "Source<#{name}> last message repeated #{count} times"
+            end
+            sleep(pause_time)
+          else
+            debug "Received: #{msgs.inspect}"
+          end
+          count += 1
+        end
+        msgs.map{|m| pre_process(m) }
+      end
+
+      def transmit(message, original=nil)
+        queue = determine_queue(original)
+        @fog.send_message(queue, message)
+      end
+
+      def confirm(message)
+        queue = determine_queue(message)
+        debug "Source<#{name}> Confirming message<#{message}> on Queue<#{queue}>"
+        m = message.is_a?(Message) ? message[:message] : message
+        @fog.delete_message(queue, m['ReceiptHandle'])
+      end
+
+      private
+
+      def determine_queue(obj)
+        queue = nil
+        if(obj)
+          if(obj.is_a?(Message))
+            queue = obj[:message]['SourceQueue']
+          else
+            case obj
+            when Numeric
+              queue = @queues[dest]
+            when String, Symbol
+              queue = @queues[dest.to_s] || queues.detect{|q| q.end_with?(dest.to_s)}
+            when Hash
+              queue = obj['SourceQueue']
+            end
+          end
+        end
+        queue || queues.first
+      end
+
+      def queues
+        if(@processable_queues)
+          @queues.map do |k,v|
+            v if @processable_queues.include?(k)
+          end.compact
+        else
+          @queues.values
+        end
+      end
+
+      def fog
+        unless(@fog)
+          connect
+        end
+        @fog
+      end
+
+      def pre_process(m)
+        if(m['Body'])
+          begin
+            m['Body'] = JSON.load(m['Body'])
+          rescue JSON::ParserError
+            # well, we did our best
+          end
+        end
+        m
+      end
+
+    end
+  end
+end

data/lib/carnivore-sqs/version.rb

@@ -0,0 +1,7 @@
+module Carnivore
+  module Sqs
+    class Version < Gem::Version
+    end
+    VERSION = Version.new('0.1.0')
+  end
+end

data/lib/carnivore-sqs.rb

@@ -0,0 +1,4 @@
+require 'carnivore-sqs/version'
+require 'carnivore'
+
+Carnivore::Source.provide(:sqs, 'carnivore-sqs/sqs')

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: carnivore-sqs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Chris Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: carnivore
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Carnivore SQS source
+email: chrisroberts.code@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/carnivore-sqs/version.rb
+- lib/carnivore-sqs/sqs.rb
+- lib/carnivore-sqs.rb
+- carnivore-sqs.gemspec
+- README.md
+- CHANGELOG.md
+homepage: https://github.com/heavywater/carnivore-sqs
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Message processing helper
+test_files: []
+has_rdoc: