carnivore 0.1.2 → 0.1.4

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# v0.1.4
+* Update `Source#send` to `Source#transmit`
+* Add common util module for logging
+* Allow SQS queues to be provided in named Hash form
+* Allow sending to named SQS queues
+* Make automatic response optional on HTTP source
+* Use `Carnivore::Message` instead of bare hash for message passing
+
 # v0.1.2
 * Allow ARN based queue endpoints for SQS
 

data/lib/carnivore/callback.rb

@@ -1,3 +1,5 @@
+require 'carnivore/utils'
+
 module Carnivore
   class Callback
 
@@ -6,6 +8,7 @@ module Carnivore
     end
 
     include Celluloid
+    include Utils::Logging
 
     attr_reader :name
 
@@ -40,21 +43,5 @@ module Carnivore
       debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
     end
 
-    # Custom logger helpers
-
-    %w(debug info warn error).each do |key|
-      define_method(key) do |string|
-        log(key, string)
-      end
-    end
-
-    def log(*args)
-      if(args.empty?)
-        Celluloid::Logger
-      else
-        severity, string = args
-        Celluloid::Logger.send(severity.to_sym, "#{self}: #{string}")
-      end
-    end
   end
 end

data/lib/carnivore/message.rb

@@ -0,0 +1,22 @@
+module Carnivore
+  class Message
+
+    attr_reader :args
+
+    def initialize(args={})
+      @args = args
+    end
+
+    def [](k)
+      @args[k.to_sym] || @args[k.to_s]
+    end
+
+    def inspect
+      "<Carnivore::Message[#{self.object_id}] @args=#{args}>"
+    end
+
+    def to_s
+      "<Carnivore::Message:#{self.object_id}>"
+    end
+  end
+end

data/lib/carnivore/source/http.rb

@@ -15,7 +15,8 @@ module Carnivore
       def default_args(args)
         {
           :bind => '0.0.0.0',
-          :port => '3000'
+          :port => '3000',
+          :auto_respond => true
         }.merge(args)
       end
 
@@ -23,13 +24,13 @@ module Carnivore
         srv = Reel::Server.supervise(args[:bind], args[:port]) do |con|
           while(req = con.request)
             begin
-              msg = format(:request => req, :body => req.body)
+              msg = format(:request => req, :body => req.body, :connection => con)
               callbacks.each do |name|
                 c_name = callback_name(name)
                 debug "Dispatching message<#{msg[:message].object_id}> to callback<#{name} (#{c_name})>"
                 Celluloid::Actor[c_name].async.call(msg)
               end
-              con.respond(:ok, 'So long, and thanks for all the fish!')
+              con.respond(:ok, 'So long, and thanks for all the fish!') if args[:auto_respond]
             rescue => e
               con.respond(:bad_request, 'Failed to process request')
             end

data/lib/carnivore/source/sqs.rb

@@ -12,13 +12,29 @@ module Carnivore
       def setup(args={})
         @fog = nil
         @connection_args = args[:fog]
-        @queues = Array(args[:queues]).compact.flatten
-        @queues.map! do |q|
-          q.include?('.com') ? q : "/#{q.split(':')[-2,2].join('/')}"
+        case args[:queues]
+        when Hash
+          @queues = args[:queues]
+        else
+          @queues = Array(args[:queues]).flatten.compact
+          @queues = Hash[*(
+              @queues.size.times.map(&:to_i).zip(@queues).flatten
+          )]
+        end
+        @queues.values.map do |q|
+          q.replace(format_queue(q))
+        end
+        if(args[:processable_queues])
+          @processable_queues = Array(args[:processable_queues]).flatten.compact
         end
         @pause_time = args[:pause] || 5
         @receive_timeout = after(args[:receive_timeout] || 30){ terminate }
         debug "Creating SQS source instance <#{name}>"
+        debug "Handled queues: #{@queues.inspect}"
+      end
+
+      def format_queue(q)
+        q.include?('.com') ? q : "/#{q.split(':')[-2,2].join('/')}"
       end
 
       def connect
@@ -31,8 +47,11 @@ module Carnivore
         while(msgs.empty?)
           msgs = []
           @receive_timeout.reset
-          msgs = @queues.map do |q|
-            @fog.receive_message(q, 'MaxNumberOfMessages' => n).body['Message']
+          msgs = queues.map do |q|
+            m = @fog.receive_message(q, 'MaxNumberOfMessages' => n).body['Message']
+            m.map! do |msg|
+              msg.merge('SourceQueue' => q)
+            end
           end.flatten.compact
           @receive_timeout.reset
           if(msgs.empty?)
@@ -42,22 +61,43 @@ module Carnivore
               debug "Source<#{name}> last message repeated #{count} times"
             end
             sleep(pause_time)
+          else
+            debug "Received: #{msgs.inspect}"
           end
           count += 1
         end
         msgs.map{|m| pre_process(m) }
       end
 
-      def send(message)
-        @fog.send_message(@queue, message)
+      def transmit(message, dest=nil)
+        case dest
+        when Numeric
+          queue = @queues[dest]
+        when String, Symbol
+          queue = @queues[dest.to_s] || @queues.detect{|q| q.include?(dest.to_s)}
+        else
+          queue = @queues.first
+        end
+        @fog.send_message(queue, message)
       end
 
       def confirm(message)
-        @fog.delete_message(@queue, message['ReceiptHandle'])
+        debug "Source<#{name}> Confirming message<#{message.object_id}>"
+        @fog.delete_message(message['SourceQueue'], message['ReceiptHandle'])
       end
 
       private
 
+      def queues
+        if(@processable_queues)
+          @queues.map do |k,v|
+            v if @processable_queues.include?(k)
+          end.compact
+        else
+          @queues.values
+        end
+      end
+
       def fog
         unless(@fog)
           connect

data/lib/carnivore/source.rb

@@ -1,4 +1,6 @@
+require 'carnivore/utils'
 require 'carnivore/callback'
+require 'carnivore/message'
 
 module Carnivore
   class Source < Celluloid::SupervisionGroup
@@ -48,16 +50,18 @@ module Carnivore
     end
 
     include Celluloid
-    include Celluloid::Logger
+    include Utils::Logging
 
     attr_reader :name
     attr_reader :callbacks
     attr_reader :auto_confirm
+    attr_reader :auto_process
     attr_reader :callback_supervisor
 
     def initialize(args={})
       @callbacks = []
       @callback_names = {}
+      @auto_process = true
       @callback_supervisor = Celluloid::SupervisionGroup.run!
       @name = args[:name] || Celluloid.uuid
       @auto_confirm = !!args[:auto_confirm]
@@ -68,7 +72,7 @@ module Carnivore
       end
       setup(args)
       connect
-      async.process
+      async.process if @auto_process
     rescue => e
       debug "Failed to initialize: #{self} - #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
       raise
@@ -82,12 +86,16 @@ module Carnivore
       "<#{self.class.name}:#{object_id} @name=#{name} @callbacks=#{Hash[*callbacks.map{|k,v| [k,v.object_id]}.flatten]}>"
     end
 
+    def to_s
+      "<#{self.class.name}:#{object_id} @name=#{name}>"
+    end
+
     def setup(args={})
-      debug "<#{self.class}> No custom setup declared"
+      debug 'No custom setup declared'
     end
 
     def connect(args={})
-      debug "<#{self.class}> No custom connect declared"
+      debug 'No custom connect declared'
     end
 
     def receive(n=1)
@@ -106,9 +114,16 @@ module Carnivore
 
     def add_callback(name, block_or_class)
       if(block_or_class.is_a?(Class))
-        debug "Adding callback class (#{block_or_class}) under supervision. Name: #{callback_name(name)}"
         size = block_or_class.workers || 1
-        @callback_supervisor.pool block_or_class, as: callback_name(name), size: size, args: [name]
+        if(size < 1)
+          warn "Callback class (#{block_or_class}) defined no workers. Skipping."
+        elsif(size == 1)
+          debug "Adding callback class (#{block_or_class}) under supervision. Name: #{callback_name(name)}"
+          @callback_supervisor.supervise_as callback_name(name), block_or_class, name
+        else
+          debug "Adding callback class (#{block_or_class}) under supervision pool (#{size} workers). Name: #{callback_name(name)}"
+          @callback_supervisor.pool block_or_class, as: callback_name(name), size: size, args: [name]
+        end
       else
         debug "Adding custom callback class  from block (#{block_or_class}) under supervision. Name: #{callback_name(name)}"
         @callback_supervisor.supervise_as callback_name(name), Callback, name, block_or_class
@@ -134,7 +149,10 @@ module Carnivore
     end
 
     def format(msg)
-      {:message => msg, :source => self}
+      Message.new(
+        :message => msg,
+        :source => self
+      )
     end
 
     def process

data/lib/carnivore/supervisor.rb

@@ -2,7 +2,11 @@ module Carnivore
   class Supervisor < Celluloid::SupervisionGroup
 
     Source.sources.each do |source|
-      supervise source.klass, as: source.source_hash[:name], args: [source.source_hash]
+      supervise(
+        source.klass,
+        as: source.source_hash[:name],
+        args: [source.source_hash]
+      )
     end
 
   end

data/lib/carnivore/utils.rb

@@ -9,10 +9,28 @@ module Carnivore
                 k.gsub(/(?<![A-Z])([A-Z])/, '_\1').sub(/^_/, '').downcase.to_sym,
                 v.is_a?(Hash) ? symbolize_hash(v) : v
               ]
-            end.flatten
+            end.flatten(1)
         )]
       end
     end
 
+    module Logging
+
+      %w(debug info warn error).each do |key|
+        define_method(key) do |string|
+          log(key, string)
+        end
+      end
+
+      def log(*args)
+        if(args.empty?)
+          Celluloid::Logger
+        else
+          severity, string = args
+          Celluloid::Logger.send(severity.to_sym, "#{self}: #{string}")
+        end
+      end
+
+    end
   end
 end

data/lib/carnivore/version.rb

@@ -1,5 +1,5 @@
 module Carnivore
   class Version < Gem::Version
   end
-  VERSION = Version.new('0.1.2')
+  VERSION = Version.new('0.1.4')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: carnivore
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-13 00:00:00.000000000 Z
+date: 2013-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -107,6 +107,7 @@ files:
 - lib/carnivore/version.rb
 - lib/carnivore/runner.rb
 - lib/carnivore/config.rb
+- lib/carnivore/message.rb
 - lib/carnivore/utils.rb
 - lib/carnivore/container.rb
 - lib/carnivore.rb