carnivore 0.1.8 → 0.1.10

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# v0.1.10
+* Remove builtin sources
+* Allow optional auto-symbolize
+* Let `confirm!` accept extra parameters
+* Add custom supervisor for future expansion
+
 # v0.1.8
 * Clean up requires
 * Register sources by name for easy lookup

data/README.md

@@ -110,5 +110,5 @@ end.start!
 
 ## Info
 
-* Repository: https://github.com/heavywater/carnivore
-* IRC: Freenode @ #heavywater
+* Repository: https://github.com/carnivore-rb/carnivore
+* IRC: Freenode @ #carnivore

data/carnivore.gemspec

@@ -6,13 +6,11 @@ Gem::Specification.new do |s|
   s.summary = 'Message processing helper'
   s.author = 'Chris Roberts'
   s.email = 'chrisroberts.code@gmail.com'
-  s.homepage = 'https://github.com/heavywater/carnivore'
+  s.homepage = 'https://github.com/carnivore-rb/carnivore'
   s.description = 'Message processing helper'
   s.require_path = 'lib'
   s.add_dependency 'fog'
   s.add_dependency 'celluloid'
-  s.add_dependency 'reel'
   s.add_dependency 'mixlib-config'
-  s.add_dependency 'bunny'
   s.files = Dir['**/*']
 end

data/lib/carnivore/config.rb

@@ -8,6 +8,13 @@ module Carnivore
 
     class << self
 
+      def auto_symbolize(v=nil)
+        unless(v.nil?)
+          @hash_symbolizer = !!v
+        end
+        @hash_symbolizer.nil? ? false : @hash_symbolizer
+      end
+
       # args:: configuration hash
       # Merge provided args into configuration
       def configure(args)
@@ -44,9 +51,14 @@ module Carnivore
       #    Config.get(:other_app, :port) => nil
       #    Config.get(:my_app, :mail, :server) => nil
       def get(*ary)
-        ary.flatten.inject(self) do |memo, key|
+        value = ary.flatten.inject(self) do |memo, key|
           memo[key.to_s] || memo[key.to_sym] || break
         end
+        if(value.is_a?(Hash) && auto_symbolize)
+          Carnivore::Utils.symbolize_hash(value)
+        else
+          value
+        end
       end
 
     end

data/lib/carnivore/message.rb

@@ -7,17 +7,21 @@ module Carnivore
 
     def initialize(args={})
       unless(args[:source])
-        raise ArgumentError.new("A valid `Carnivore::Source` must be provided via `:source`")
+        raise ArgumentError.new("A valid `Carnivore::Source` name must be provided via `:source`")
       end
       @args = args.dup
     end
 
     def [](k)
-      @args[k.to_sym] || @args[k.to_s]
+      if(k.to_sym == :source)
+        Celluloid::Actor[@args[:source]]
+      else
+        @args[k.to_sym] || @args[k.to_s]
+      end
     end
 
-    def confirm!
-      self[:source].confirm(self)
+    def confirm!(*args)
+      self[:source].confirm(*([self] + args))
     end
 
     def inspect

data/lib/carnivore/runner.rb

@@ -12,10 +12,20 @@ module Carnivore
     end
 
     def start!
+      supervisor = nil
       begin
         require 'carnivore/supervisor'
-        Supervisor.run
+        supervisor = Carnivore::Supervisor.run!
+        Source.sources.each do |source|
+          supervisor.supervise_as(
+            source.source_hash[:name],
+            source.klass,
+            source.source_hash
+          )
+        end
+        supervisor.wait(:kill_all_humans)
       rescue Exception => e
+        supervisor.terminate
         # Gracefully shut down
       end
     end

data/lib/carnivore/source.rb

@@ -91,12 +91,14 @@ module Carnivore
     attr_reader :callbacks
     attr_reader :auto_confirm
     attr_reader :auto_process
+    attr_reader :run_process
     attr_reader :callback_supervisor
 
     def initialize(args={})
       @callbacks = []
       @callback_names = {}
       @auto_process = args.fetch(:auto_process, true)
+      @run_process = @auto_process
       @auto_confirm = !!args[:auto_confirm]
       @callback_supervisor = Celluloid::SupervisionGroup.run!
       @name = args[:name] || Celluloid.uuid
@@ -145,10 +147,6 @@ module Carnivore
       debug 'No custom confirm declared'
     end
 
-    def terminate
-      @callback_supervisor.finalize
-    end
-
     def add_callback(name, block_or_class)
       if(block_or_class.is_a?(Class))
         size = block_or_class.workers || 1
@@ -189,12 +187,12 @@ module Carnivore
     def format(msg)
       Message.new(
         :message => msg,
-        :source => self
+        :source => self.name
       )
     end
 
     def process
-      loop do
+      while(run_process)
         msgs = Array(receive).flatten.compact.map do |m|
           format(m)
         end

data/lib/carnivore/supervisor.rb

@@ -1,13 +1,4 @@
 module Carnivore
   class Supervisor < Celluloid::SupervisionGroup
-
-    Source.sources.each do |source|
-      supervise(
-        source.klass,
-        as: source.source_hash[:name],
-        args: [source.source_hash]
-      )
-    end
-
   end
 end

data/lib/carnivore/utils.rb

@@ -21,6 +21,8 @@ module Carnivore
       end
     end
 
+    extend Params
+
     module Logging
 
       %w(debug info warn error).each do |key|
@@ -39,5 +41,8 @@ module Carnivore
       end
 
     end
+
+    extend Logging
+
   end
 end

data/lib/carnivore/version.rb

@@ -1,5 +1,5 @@
 module Carnivore
   class Version < Gem::Version
   end
-  VERSION = Version.new('0.1.8')
+  VERSION = Version.new('0.1.10')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: carnivore
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-18 00:00:00.000000000 Z
+date: 2013-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -43,22 +43,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: reel
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: mixlib-config
   requirement: !ruby/object:Gem::Requirement
@@ -75,22 +59,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bunny
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Message processing helper
 email: chrisroberts.code@gmail.com
 executables: []
@@ -99,10 +67,7 @@ extra_rdoc_files: []
 files:
 - lib/carnivore/callback.rb
 - lib/carnivore/supervisor.rb
-- lib/carnivore/source/http.rb
-- lib/carnivore/source/sqs.rb
 - lib/carnivore/source/test.rb
-- lib/carnivore/source/rabbitmq.rb
 - lib/carnivore/source.rb
 - lib/carnivore/version.rb
 - lib/carnivore/runner.rb
@@ -122,10 +87,9 @@ files:
 - examples/test_block.rb
 - Gemfile
 - README.md
-- carnivore-0.1.6.gem
 - CHANGELOG.md
 - carnivore.gemspec
-homepage: https://github.com/heavywater/carnivore
+homepage: https://github.com/carnivore-rb/carnivore
 licenses: []
 post_install_message: 
 rdoc_options: []

data/lib/carnivore/source/http.rb

@@ -1,44 +0,0 @@
-require 'reel'
-require 'carnivore/source'
-
-module Carnivore
-  class Source
-
-    class Http < Source
-
-      attr_reader :args
-
-      def setup(args={})
-        @args = default_args(args)
-      end
-
-      def default_args(args)
-        {
-          :bind => '0.0.0.0',
-          :port => '3000',
-          :auto_respond => true
-        }.merge(args)
-      end
-
-      def process(*process_args)
-        srv = Reel::Server.supervise(args[:bind], args[:port]) do |con|
-          while(req = con.request)
-            begin
-              msg = format(:request => req, :body => req.body, :connection => con)
-              callbacks.each do |name|
-                c_name = callback_name(name)
-                debug "Dispatching message<#{msg[:message].object_id}> to callback<#{name} (#{c_name})>"
-                Celluloid::Actor[c_name].async.call(msg)
-              end
-              con.respond(:ok, 'So long, and thanks for all the fish!') if args[:auto_respond]
-            rescue => e
-              con.respond(:bad_request, 'Failed to process request')
-            end
-          end
-        end
-      end
-
-    end
-
-  end
-end

data/lib/carnivore/source/rabbitmq.rb

@@ -1,37 +0,0 @@
-require 'fog'
-require 'carnivore/source'
-
-module Carnivore
-  class Source
-    class RabbitMq < Source
-
-      def setup(args={})
-        @bunny = nil
-        @connection_args = args[:bunny]
-        @queue_name = args[:queue]
-        @exchange_name = args[:exchange]
-        debug "Creating RabbitMq source instance <#{name}>"
-      end
-
-      def connect
-        @bunny = Bunny.new(@connection_args)
-        @bunny.start
-        @channel = @bunny.create_channel
-        @exchange = @channel.topic(@exchange_name)
-        @queue = @channel.queue(@queue_name).bind(@exchange) # TODO: Add topic key
-      end
-
-      def process
-        @queue.subscribe do |info, metadata, payload|
-          msg = format(payload)
-          callbacks.each do |name|
-            c_name = callback_name(name)
-            debug "Dispatching message<#{msg[:message].object_id}> to callback<#{name} (#{c_name})>"
-            Celluloid::Actor[c_name].async.call(msg)
-          end
-        end
-      end
-
-    end
-  end
-end

data/lib/carnivore/source/sqs.rb

@@ -1,135 +0,0 @@
-require 'fog'
-require 'carnivore/source'
-
-module Carnivore
-  class Source
-    class Sqs < Source
-
-      OUTPUT_REPEAT_EVERY=20
-
-      attr_reader :pause_time
-
-      def setup(args={})
-        @fog = nil
-        @connection_args = args[:fog]
-        case args[:queues]
-        when Hash
-          @queues = args[:queues]
-        else
-          @queues = Array(args[:queues]).flatten.compact
-          @queues = Hash[*(
-              @queues.size.times.map(&:to_i).zip(@queues).flatten
-          )]
-        end
-        @queues.values.map do |q|
-          q.replace(format_queue(q))
-        end
-        if(args[:processable_queues])
-          @processable_queues = Array(args[:processable_queues]).flatten.compact
-        end
-        @pause_time = args[:pause] || 5
-        @receive_timeout = after(args[:receive_timeout] || 30){ terminate }
-        debug "Creating SQS source instance <#{name}>"
-        debug "Handled queues: #{@queues.inspect}"
-      end
-
-      def format_queue(q)
-        q.include?('.com') ? q : "/#{q.split(':')[-2,2].join('/')}"
-      end
-
-      def connect
-        @fog = Fog::AWS::SQS.new(@connection_args)
-      end
-
-      def receive(n=1)
-        count = 0
-        msgs = []
-        while(msgs.empty?)
-          msgs = []
-          @receive_timeout.reset
-          msgs = queues.map do |q|
-            m = @fog.receive_message(q, 'MaxNumberOfMessages' => n).body['Message']
-            m.map! do |msg|
-              msg.merge('SourceQueue' => q)
-            end
-          end.flatten.compact
-          @receive_timeout.reset
-          if(msgs.empty?)
-            if(count == 0)
-              debug "Source<#{name}> no message received. Sleeping for #{pause_time} seconds."
-            elsif(count % OUTPUT_REPEAT_EVERY == 0)
-              debug "Source<#{name}> last message repeated #{count} times"
-            end
-            sleep(pause_time)
-          else
-            debug "Received: #{msgs.inspect}"
-          end
-          count += 1
-        end
-        msgs.map{|m| pre_process(m) }
-      end
-
-      def transmit(message, original=nil)
-        queue = determine_queue(original)
-        @fog.send_message(queue, message)
-      end
-
-      def confirm(message)
-        queue = determine_queue(message)
-        debug "Source<#{name}> Confirming message<#{message}> on Queue<#{queue}>"
-        m = message.is_a?(Message) ? message[:message] : message
-        @fog.delete_message(queue, m['ReceiptHandle'])
-      end
-
-      private
-
-      def determine_queue(obj)
-        queue = nil
-        if(obj)
-          if(obj.is_a?(Message))
-            queue = obj[:message]['SourceQueue']
-          else
-            case obj
-            when Numeric
-              queue = @queues[dest]
-            when String, Symbol
-              queue = @queues[dest.to_s] || queues.detect{|q| q.end_with?(dest.to_s)}
-            when Hash
-              queue = obj['SourceQueue']
-            end
-          end
-        end
-        queue || queues.first
-      end
-
-      def queues
-        if(@processable_queues)
-          @queues.map do |k,v|
-            v if @processable_queues.include?(k)
-          end.compact
-        else
-          @queues.values
-        end
-      end
-
-      def fog
-        unless(@fog)
-          connect
-        end
-        @fog
-      end
-
-      def pre_process(m)
-        if(m['Body'])
-          begin
-            m['Body'] = JSON.load(m['Body'])
-          rescue JSON::ParserError
-            # well, we did our best
-          end
-        end
-        m
-      end
-
-    end
-  end
-end